diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 3fb45794513f0f362cfcf26bb809dd5d059b1d58..73bc748705c49700d973b641dd0fd5a4534980fc 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -68,6 +68,14 @@ typedef uint16_t tmsg_t; #define TSDB_IE_TYPE_DNODE_EXT 6 #define TSDB_IE_TYPE_DNODE_STATE 7 +typedef enum { + HEARTBEAT_TYPE_MQ = 0, + HEARTBEAT_TYPE_QUERY = 1, + // types can be added here + // + HEARTBEAT_TYPE_MAX +} EHbType; + typedef enum _mgmt_table { TSDB_MGMT_TABLE_START, TSDB_MGMT_TABLE_ACCT, @@ -147,7 +155,7 @@ typedef struct { typedef struct { SClientHbKey connKey; - SHashObj* info; // hash + SHashObj* info; // hash } SClientHbReq; typedef struct { @@ -173,7 +181,10 @@ static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { } int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq); -void* tDeserializeClientHbReq(void* buf, SClientHbReq* pReq); +void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq); + +int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp); +void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp); static FORCE_INLINE void tFreeClientHbReq(void *pReq) { SClientHbReq* req = (SClientHbReq*)pReq; @@ -182,14 +193,17 @@ static FORCE_INLINE void tFreeClientHbReq(void *pReq) { } int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq); -void* tDeserializeClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); +void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq) { SClientHbBatchReq *req = (SClientHbBatchReq*)pReq; - taosArrayDestroyEx(req->reqs, tFreeClientHbReq); + //taosArrayDestroyEx(req->reqs, tFreeClientHbReq); free(pReq); } +int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp); +void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp); + static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) { int tlen = 0; tlen += taosEncodeFixedI32(buf, pKv->keyLen); @@ -220,6 +234,7 @@ static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) return buf; } + typedef struct { int32_t vgId; char* dbName; @@ -359,6 +374,31 @@ static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) { return buf; } +typedef struct SMqHbRsp { + int8_t status; //idle or not + int8_t vnodeChanged; + int8_t epChanged; // should use new epset + int8_t reserved; + SEpSet epSet; +} SMqHbRsp; + +static FORCE_INLINE int taosEncodeSMqHbRsp(void** buf, const SMqHbRsp* pRsp) { + int tlen = 0; + tlen += taosEncodeFixedI8(buf, pRsp->status); + tlen += taosEncodeFixedI8(buf, pRsp->vnodeChanged); + tlen += taosEncodeFixedI8(buf, pRsp->epChanged); + tlen += taosEncodeSEpSet(buf, &pRsp->epSet); + return tlen; +} + +static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) { + buf = taosDecodeFixedI8(buf, &pRsp->status); + buf = taosDecodeFixedI8(buf, &pRsp->vnodeChanged); + buf = taosDecodeFixedI8(buf, &pRsp->epChanged); + buf = taosDecodeSEpSet(buf, &pRsp->epSet); + return buf; +} + typedef struct { int32_t acctId; int64_t clusterId; @@ -993,6 +1033,13 @@ typedef struct { uint64_t taskId; } SSinkDataReq; +typedef struct { + SMsgHead header; + uint64_t sId; + uint64_t queryId; + uint64_t taskId; +} SQueryContinueReq; + typedef struct { SMsgHead header; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 592672b32b0d668e064b87d5c9003bb96b157733..bfeba885d0ddb6bd3ffee645353d9ac90b1f48a1 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -129,7 +129,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_VGROUP_LIST, "mnode-vgroup-list", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "mnode-kill-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "mnode-kill-conn", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "mnode-heartbeat", SClientHbBatchReq, SClientHbBatchRsp) TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "mnode-show", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SHOW_RETRIEVE, "mnode-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "mnode-status", NULL, NULL) @@ -170,6 +170,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES, "vnode-show-tables", SVShowTablesReq, SVShowTablesRsp) TD_DEF_MSG_TYPE(TDMT_VND_SHOW_TABLES_FETCH, "vnode-show-tables-fetch", SVShowTablesFetchReq, SVShowTablesFetchRsp) + TD_DEF_MSG_TYPE(TDMT_VND_QUERY_CONTINUE, "vnode-query-continue", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_SCHEDULE_DATA_SINK, "vnode-schedule-data-sink", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 9897467230675dae1b4b3e7909acd748504dcec7..08b5fb98e7d8e723d97381e9b511717e51355a4d 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -22,9 +22,18 @@ extern "C" { #include "trpc.h" + +enum { + NODE_TYPE_VNODE = 1, + NODE_TYPE_QNODE, + NODE_TYPE_SNODE, +}; + + + typedef struct SQWorkerCfg { uint32_t maxSchedulerNum; - uint32_t maxResCacheNum; + uint32_t maxTaskNum; uint32_t maxSchTaskNum; } SQWorkerCfg; @@ -39,11 +48,17 @@ typedef struct { uint64_t numOfErrors; } SQWorkerStat; +typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *); -int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt); + +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp); int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); +int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); + +int32_t qWorkerProcessDataSinkMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); + int32_t qWorkerProcessReadyMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); int32_t qWorkerProcessStatusMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); diff --git a/source/client/inc/clientHb.h b/source/client/inc/clientHb.h deleted file mode 100644 index 7bc4311b29dc5c89203b3b32054ec7186dfee073..0000000000000000000000000000000000000000 --- a/source/client/inc/clientHb.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tarray.h" -#include "thash.h" -#include "tmsg.h" - -#define HEARTBEAT_INTERVAL 1500 // ms - -typedef enum { - HEARTBEAT_TYPE_MQ = 0, - // types can be added here - // - HEARTBEAT_TYPE_MAX -} EHbType; - -typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq); - -typedef struct SAppHbMgr { - // statistics - int32_t reportCnt; - int32_t connKeyCnt; - int64_t reportBytes; // not implemented - int64_t startTime; - // ctl - SRWLatch lock; // lock is used in serialization - // connection - void* transporter; - SEpSet epSet; - // info - SHashObj* activeInfo; // hash - SHashObj* getInfoFuncs; // hash -} SAppHbMgr; - -typedef struct SClientHbMgr { - int8_t inited; - // ctl - int8_t threadStop; - pthread_t thread; - pthread_mutex_t lock; // used when app init and cleanup - SArray* appHbMgrs; // SArray one for each cluster - FHbRspHandle handle[HEARTBEAT_TYPE_MAX]; -} SClientHbMgr; - -// TODO: embed param into function -// return type: SArray -typedef SArray* (*FGetConnInfo)(SClientHbKey connKey, void* param); - -// global, called by mgmt -int hbMgrInit(); -void hbMgrCleanUp(); -int hbHandleRsp(SClientHbBatchRsp* hbRsp); - -// cluster level -SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet); -void appHbMgrCleanup(SAppHbMgr* pAppHbMgr); - -// conn level -int hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func); -void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); - -int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); - -// mq -void hbMgrInitMqHbRspHandle(); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 8b530ab6cfa7346c05492139e041e1194330bd08..b6f38b12cac7feeaf0da1b2baba90e20f1414c4b 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -31,6 +31,41 @@ extern "C" { #include "trpc.h" #include "query.h" +#define HEARTBEAT_INTERVAL 1500 // ms + +typedef struct SAppInstInfo SAppInstInfo; + +typedef int32_t (*FHbRspHandle)(SClientHbRsp* pReq); + +typedef struct SAppHbMgr { + // statistics + int32_t reportCnt; + int32_t connKeyCnt; + int64_t reportBytes; // not implemented + int64_t startTime; + // ctl + SRWLatch lock; // lock is used in serialization + // connection + SAppInstInfo* pAppInstInfo; + // info + SHashObj* activeInfo; // hash + SHashObj* getInfoFuncs; // hash +} SAppHbMgr; + +typedef struct SClientHbMgr { + int8_t inited; + // ctl + int8_t threadStop; + pthread_t thread; + pthread_mutex_t lock; // used when app init and cleanup + SArray* appHbMgrs; // SArray one for each cluster + FHbRspHandle handle[HEARTBEAT_TYPE_MAX]; +} SClientHbMgr; + +// TODO: embed param into function +// return type: SArray +typedef SArray* (*FGetConnInfo)(SClientHbKey connKey, void* param); + typedef struct SQueryExecMetric { int64_t start; // start timestamp int64_t parsed; // start to parse @@ -55,15 +90,15 @@ typedef struct SHeartBeatInfo { void *pTimer; // timer, used to send request msg to mnode } SHeartBeatInfo; -typedef struct SAppInstInfo { - int64_t numOfConns; - SCorEpSet mgmtEp; - SInstanceSummary summary; +struct SAppInstInfo { + int64_t numOfConns; + SCorEpSet mgmtEp; + SInstanceSummary summary; SList *pConnList; // STscObj linked list - int64_t clusterId; + int64_t clusterId; void *pTransporter; - SHeartBeatInfo hb; -} SAppInstInfo; + struct SAppHbMgr *pAppHbMgr; +}; typedef struct SAppInfo { int64_t startTime; @@ -81,12 +116,17 @@ typedef struct STscObj { char db[TSDB_DB_FNAME_LEN]; int32_t acctId; uint32_t connId; + int32_t connType; uint64_t id; // ref ID returned by taosAddRef pthread_mutex_t mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo *pAppInfo; } STscObj; +typedef struct SMqConsumer { + STscObj* pTscObj; +} SMqConsumer; + typedef struct SReqResultInfo { const char *pRspMsg; const char *pData; @@ -169,6 +209,26 @@ void *doFetchRow(SRequestObj* pRequest); void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); +// --- heartbeat +// global, called by mgmt +int hbMgrInit(); +void hbMgrCleanUp(); +int hbHandleRsp(SClientHbBatchRsp* hbRsp); + +// cluster level +SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo); +void appHbMgrCleanup(SAppHbMgr* pAppHbMgr); + +// conn level +int hbRegisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, FGetConnInfo func); +void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); + +int hbAddConnInfo(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* key, void* value, int32_t keyLen, int32_t valueLen); + +// --- mq +void hbMgrInitMqHbRspHandle(); + + #ifdef __cplusplus } #endif diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 9bbd62c1d9bfcdf095c0fd15140e2f644cb756b8..6d7fc9f81a1de3effef84893bc1469afdca40732 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "clientHb.h" +#include "clientInt.h" #include "trpc.h" static SClientHbMgr clientHbMgr = {0}; @@ -21,10 +21,18 @@ static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); static void hbStopThread(); -static int32_t hbMqHbRspHandle(SClientHbRsp* pReq) { +static int32_t hbMqHbRspHandle(SClientHbRsp* pRsp) { return 0; } +static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code) { + if (code != 0) { + return -1; + } + SClientHbRsp* pRsp = (SClientHbRsp*) pMsg->pData; + return hbMqHbRspHandle(pRsp); +} + void hbMgrInitMqHbRspHandle() { clientHbMgr.handle[HEARTBEAT_TYPE_MQ] = hbMqHbRspHandle; } @@ -35,18 +43,18 @@ static FORCE_INLINE void hbMgrInitHandle() { } SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { - SClientHbBatchReq* pReq = malloc(sizeof(SClientHbBatchReq)); - if (pReq == NULL) { + SClientHbBatchReq* pBatchReq = malloc(sizeof(SClientHbBatchReq)); + if (pBatchReq == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); - pReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); + pBatchReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); void *pIter = taosHashIterate(pAppHbMgr->activeInfo, NULL); while (pIter != NULL) { - taosArrayPush(pReq->reqs, pIter); SClientHbReq* pOneReq = pIter; + taosArrayPush(pBatchReq->reqs, pOneReq); taosHashClear(pOneReq->info); pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); @@ -59,10 +67,10 @@ SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { taosHashCopyKey(pIter, &connKey); getConnInfoFp(connKey, NULL); - pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); + pIter = taosHashIterate(pAppHbMgr->getInfoFuncs, pIter); } - return pReq; + return pBatchReq; } static void* hbThreadFunc(void* param) { @@ -75,20 +83,48 @@ static void* hbThreadFunc(void* param) { int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for(int i = 0; i < sz; i++) { - SAppHbMgr* pAppHbMgr = taosArrayGet(clientHbMgr.appHbMgrs, i); - SClientHbBatchReq* pReq = hbGatherAllInfo(pAppHbMgr); - void* reqStr = NULL; - int tlen = tSerializeSClientHbBatchReq(&reqStr, pReq); - SMsgSendInfo info; - /*info.fp = hbHandleRsp;*/ + SAppHbMgr* pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); + int32_t connCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); + if (connCnt == 0) { + continue; + } + SClientHbBatchReq* pReq = hbGatherAllInfo(pAppHbMgr); + if (pReq == NULL) { + continue; + } + int tlen = tSerializeSClientHbBatchReq(NULL, pReq); + void *buf = malloc(tlen); + if (buf == NULL) { + //TODO: error handling + break; + } + void *bufCopy = buf; + tSerializeSClientHbBatchReq(&bufCopy, pReq); + SMsgSendInfo *pInfo = malloc(sizeof(SMsgSendInfo)); + if (pInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tFreeClientHbBatchReq(pReq); + free(buf); + break; + } + pInfo->fp = hbMqAsyncCallBack; + pInfo->msgInfo.pData = buf; + pInfo->msgInfo.len = tlen; + pInfo->msgType = TDMT_MND_HEARTBEAT; + pInfo->param = NULL; + pInfo->requestId = generateRequestId(); + pInfo->requestObjRefId = 0; + + SAppInstInfo *pAppInstInfo = pAppHbMgr->pAppInstInfo; int64_t transporterId = 0; - asyncSendMsgToServer(pAppHbMgr->transporter, &pAppHbMgr->epSet, &transporterId, &info); + SEpSet epSet = getEpSet_s(&pAppInstInfo->mgmtEp); + asyncSendMsgToServer(pAppInstInfo->pTransporter, &epSet, &transporterId, pInfo); tFreeClientHbBatchReq(pReq); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); - taosMsleep(HEARTBEAT_INTERVAL); } + taosMsleep(HEARTBEAT_INTERVAL); } return NULL; } @@ -110,7 +146,8 @@ static void hbStopThread() { atomic_store_8(&clientHbMgr.threadStop, 1); } -SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet) { +SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo) { + hbMgrInit(); SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -119,16 +156,27 @@ SAppHbMgr* appHbMgrInit(void* transporter, SEpSet epSet) { // init stat pAppHbMgr->startTime = taosGetTimestampMs(); - // init connection info - pAppHbMgr->transporter = transporter; - pAppHbMgr->epSet = epSet; + // init app info + pAppHbMgr->pAppInstInfo = pAppInstInfo; // init hash info pAppHbMgr->activeInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + + if (pAppHbMgr->activeInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + free(pAppHbMgr); + return NULL; + } pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq; // init getInfoFunc pAppHbMgr->getInfoFuncs = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + if (pAppHbMgr->getInfoFuncs == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + free(pAppHbMgr); + return NULL; + } + taosArrayPush(clientHbMgr.appHbMgrs, &pAppHbMgr); return pAppHbMgr; } @@ -138,7 +186,7 @@ void appHbMgrCleanup(SAppHbMgr* pAppHbMgr) { int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { - SAppHbMgr* pTarget = taosArrayGet(clientHbMgr.appHbMgrs, i); + SAppHbMgr* pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); if (pAppHbMgr == pTarget) { taosHashCleanup(pTarget->activeInfo); taosHashCleanup(pTarget->getInfoFuncs); @@ -171,7 +219,6 @@ void hbMgrCleanUp() { if (old == 0) return; taosArrayDestroy(clientHbMgr.appHbMgrs); - } int hbHandleRsp(SClientHbBatchRsp* hbRsp) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 26d54c138759a5edcdf24c04ff4974ccf4120d15..e2143d20df976c35675c031a8ad4ff8d51f14bd5 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -117,6 +117,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); + p->pAppHbMgr = appHbMgrInit(p); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; @@ -259,6 +260,101 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { return scheduleAsyncExecJob(pRequest->pTscObj->pAppInfo->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); } +typedef struct tmq_t tmq_t; + +typedef struct SMqClientTopic { + // subscribe info + int32_t sqlLen; + char* sql; + char* topicName; + int64_t topicId; + // statistics + int64_t consumeCnt; + // offset + int64_t committedOffset; + int64_t currentOffset; + //connection info + int32_t vgId; + SEpSet epSet; +} SMqClientTopic; + +typedef struct tmq_resp_err_t { + int32_t code; +} tmq_resp_err_t; + +typedef struct tmq_topic_vgroup_list_t { + char* topicName; + int32_t vgId; + int64_t committedOffset; +} tmq_topic_vgroup_list_t; + +typedef void (tmq_commit_cb(tmq_t*, tmq_resp_err_t, tmq_topic_vgroup_list_t*, void* param)); + +typedef struct tmq_conf_t{ + char* clientId; + char* groupId; + char* ip; + uint16_t port; + tmq_commit_cb* commit_cb; +} tmq_conf_t; + +struct tmq_t { + char groupId[256]; + char clientId[256]; + STscObj* pTscObj; + tmq_commit_cb* commit_cb; + SArray* clientTopics; // SArray +}; + +void tmq_conf_set_offset_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb) { + conf->commit_cb = cb; +} + +SArray* tmqGetConnInfo(SClientHbKey connKey, void* param) { + tmq_t* pTmq = (void*)param; + SArray* pArray = taosArrayInit(0, sizeof(SKv)); + if (pArray == NULL) { + return NULL; + } + SKv kv = {0}; + kv.key = malloc(256); + if (kv.key == NULL) { + taosArrayDestroy(pArray); + return NULL; + } + strcpy(kv.key, "groupId"); + kv.keyLen = strlen("groupId") + 1; + kv.value = malloc(256); + if (kv.value == NULL) { + free(kv.key); + taosArrayDestroy(pArray); + return NULL; + } + strcpy(kv.value, pTmq->groupId); + kv.valueLen = strlen(pTmq->groupId) + 1; + + taosArrayPush(pArray, &kv); + strcpy(kv.key, "clientUid"); + kv.keyLen = strlen("clientUid") + 1; + *(uint32_t*)kv.value = pTmq->pTscObj->connId; + kv.valueLen = sizeof(uint32_t); + + return NULL; +} + +tmq_t* tmqCreateConsumerImpl(TAOS* conn, tmq_conf_t* conf) { + tmq_t* pTmq = malloc(sizeof(tmq_t)); + if (pTmq == NULL) { + return NULL; + } + strcpy(pTmq->groupId, conf->groupId); + strcpy(pTmq->clientId, conf->clientId); + pTmq->pTscObj = (STscObj*)conn; + pTmq->pTscObj->connType = HEARTBEAT_TYPE_MQ; + + return pTmq; +} + TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, int sqlLen) { STscObj *pTscObj = (STscObj*)taos; SRequestObj *pRequest = NULL; @@ -351,6 +447,25 @@ _return: return pRequest; } +typedef struct tmq_message_t { + int32_t numOfRows; + char* topicName; + TAOS_ROW row[]; +} tmq_message_t; + +tmq_message_t* tmq_consume_poll(tmq_t* mq, int64_t blocking_time) { + return NULL; +} + +tmq_resp_err_t* tmq_commit(tmq_t* mq, void* callback, int32_t async) { + return NULL; +} + +void tmq_message_destroy(tmq_message_t* mq_message) { + +} + + TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { STscObj *pTscObj = (STscObj *)taos; if (sqlLen > (size_t) tsMaxSQLStringLen) { @@ -708,4 +823,4 @@ void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* p pResultInfo->completed = (pRsp->completed == 1); setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); -} \ No newline at end of file +} diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 4f1a96ad29d90f9ab3b3c8e1d0e5b6493adaf338..497ef1ac95d01f426a34bfd3974c3737c9bdd8c7 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -71,6 +71,9 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { pTscObj->pAppInfo->clusterId = pConnect->clusterId; atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); + SClientHbKey connKey = {.connId = pConnect->connId, .hbType = HEARTBEAT_TYPE_QUERY}; + hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, NULL); + // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, pTscObj->pAppInfo->numOfConns); @@ -379,4 +382,4 @@ void initMsgHandleFp() { handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = processShowRsp; handleRequestRspFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = processRetrieveVndRsp; -} \ No newline at end of file +} diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index db2881ccc191cc92e5f2bf2f7a600ef547a6375d..9810b308574eacf78e29e3104bbab49b856532c4 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -48,13 +48,14 @@ int main(int argc, char** argv) { TEST(testCase, driverInit_Test) { taos_init(); } -//TEST(testCase, connect_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// if (pConn == NULL) { -// printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); -// } -// taos_close(pConn); -//} +TEST(testCase, connect_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + printf("failed to connect to server, reason:%s\n", taos_errstr(NULL)); + } + sleep(3); + taos_close(pConn); +} //TEST(testCase, create_user_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -148,36 +149,27 @@ TEST(testCase, driverInit_Test) { taos_init(); } //} // //TEST(testCase, create_db_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop database, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create database abc1 vgroups 2"); -// if (taos_errno(pRes) != 0) { -// printf("error in create db, reason:%s\n", taos_errstr(pRes)); -// } -// -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// ASSERT_TRUE(pFields == NULL); -// -// int32_t numOfFields = taos_num_fields(pRes); -// ASSERT_EQ(numOfFields, 0); -// -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create database if not exists abc1 vgroups 4"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create database abc1, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); + + //TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); + //if (taos_errno(pRes) != 0) { + //printf("error in create db, reason:%s\n", taos_errstr(pRes)); + //} + + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //ASSERT_TRUE(pFields == NULL); + + //int32_t numOfFields = taos_num_fields(pRes); + //ASSERT_EQ(numOfFields, 0); + + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "create database abc1 vgroups 4"); + //if (taos_errno(pRes) != 0) { + //printf("error in create db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_close(pConn); //} // //TEST(testCase, create_dnode_Test) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 53f59c7d57f4db395bdef1d2ac1ca5e0ef698b25..4d1a07be217fb942317f095cebec5091835feb11 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -89,7 +89,7 @@ int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { int tlen = 0; tlen += taosEncodeSClientHbKey(buf, &pReq->connKey); - int kvNum = taosHashGetSize(pReq->info); + int32_t kvNum = taosHashGetSize(pReq->info); tlen += taosEncodeFixedI32(buf, kvNum); SKv kv; void* pIter = taosHashIterate(pReq->info, pIter); @@ -104,14 +104,15 @@ int tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { return tlen; } -void *tDeserializeClientHbReq(void *buf, SClientHbReq *pReq) { - ASSERT(pReq->info != NULL); +void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) { buf = taosDecodeSClientHbKey(buf, &pReq->connKey); // TODO: error handling - int kvNum; - taosDecodeFixedI32(buf, &kvNum); - pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + int32_t kvNum; + buf = taosDecodeFixedI32(buf, &kvNum); + if (pReq->info == NULL) { + pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + } for(int i = 0; i < kvNum; i++) { SKv kv; buf = taosDecodeSKv(buf, &kv); @@ -121,12 +122,69 @@ void *tDeserializeClientHbReq(void *buf, SClientHbReq *pReq) { return buf; } -int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq) { +int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp) { + int tlen = 0; + tlen += taosEncodeSClientHbKey(buf, &pRsp->connKey); + tlen += taosEncodeFixedI32(buf, pRsp->status); + tlen += taosEncodeFixedI32(buf, pRsp->bodyLen); + tlen += taosEncodeBinary(buf, pRsp->body, pRsp->bodyLen); + return tlen; +} +void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp) { + buf = taosDecodeSClientHbKey(buf, &pRsp->connKey); + buf = taosDecodeFixedI32(buf, &pRsp->status); + buf = taosDecodeFixedI32(buf, &pRsp->bodyLen); + buf = taosDecodeBinary(buf, &pRsp->body, pRsp->bodyLen); + return buf; +} + +int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pBatchReq) { int tlen = 0; + tlen += taosEncodeFixedI64(buf, pBatchReq->reqId); + int32_t reqNum = taosArrayGetSize(pBatchReq->reqs); + tlen += taosEncodeFixedI32(buf, reqNum); + for (int i = 0; i < reqNum; i++) { + SClientHbReq* pReq = taosArrayGet(pBatchReq->reqs, i); + tlen += tSerializeSClientHbReq(buf, pReq); + } return tlen; } -void* tDeserializeClientHbBatchReq(void* buf, SClientHbBatchReq* pReq) { +void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pBatchReq) { + buf = taosDecodeFixedI64(buf, &pBatchReq->reqId); + if (pBatchReq->reqs == NULL) { + pBatchReq->reqs = taosArrayInit(0, sizeof(SClientHbReq)); + } + int32_t reqNum; + buf = taosDecodeFixedI32(buf, &reqNum); + for (int i = 0; i < reqNum; i++) { + SClientHbReq req = {0}; + buf = tDeserializeSClientHbReq(buf, &req); + taosArrayPush(pBatchReq->reqs, &req); + } + return buf; +} + +int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp) { + int tlen = 0; + int32_t sz = taosArrayGetSize(pBatchRsp->rsps); + tlen += taosEncodeFixedI32(buf, sz); + for (int i = 0; i < sz; i++) { + SClientHbRsp* pRsp = taosArrayGet(pBatchRsp->rsps, i); + tlen += tSerializeSClientHbRsp(buf, pRsp); + } + return tlen; +} + +void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp) { + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pBatchRsp->rsps = taosArrayInit(sz, sizeof(SClientHbRsp)); + for (int i = 0; i < sz; i++) { + SClientHbRsp rsp = {0}; + buf = tDeserializeSClientHbRsp(buf, &rsp); + taosArrayPush(pBatchRsp->rsps, &rsp); + } return buf; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index fced3facbec8443552b3ec5f90803f23a0da7f7b..79e5d9eae595eb95be701c3f0da8518cbd55f660 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -258,6 +258,39 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { } static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + char *batchReqStr = pReq->rpcMsg.pCont; + SClientHbBatchReq batchReq = {0}; + tDeserializeSClientHbBatchReq(batchReqStr, &batchReq); + SArray *pArray = batchReq.reqs; + int sz = taosArrayGetSize(pArray); + + SClientHbBatchRsp batchRsp = {0}; + batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); + + for (int i = 0; i < sz; i++) { + SClientHbReq* pHbReq = taosArrayGet(pArray, i); + if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_QUERY) { + + } else if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_MQ) { + SClientHbRsp rsp = { + .status = 0, + .connKey = pHbReq->connKey, + .bodyLen = 0, + .body = NULL + }; + taosArrayPush(batchRsp.rsps, &rsp); + } + } + int32_t tlen = tSerializeSClientHbBatchRsp(NULL, &batchRsp); + void* buf = rpcMallocCont(tlen); + void* bufCopy = buf; + tSerializeSClientHbBatchRsp(&bufCopy, &batchRsp); + pReq->contLen = tlen; + pReq->pCont = buf; + return 0; + +#if 0 SMnode *pMnode = pReq->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -327,6 +360,7 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { pReq->contLen = sizeof(SConnectRsp); pReq->pCont = pRsp; return 0; +#endif } static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) { diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index bf047517d3bbb47b11ca5582de1660c46bdf597b..4b329886eba63cfd4a1864a4796c73db604fbef0 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -96,6 +96,38 @@ TEST_F(MndTestProfile, 03_ConnectMsg_Show) { } TEST_F(MndTestProfile, 04_HeartBeatMsg) { + SClientHbBatchReq batchReq; + batchReq.reqs = taosArrayInit(0, sizeof(SClientHbReq)); + SClientHbReq req = {0}; + req.connKey = {.connId = 123, .hbType = HEARTBEAT_TYPE_MQ}; + req.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + SKv kv; + kv.key = (void*)"abc"; + kv.keyLen = 4; + kv.value = (void*)"bcd"; + kv.valueLen = 4; + taosHashPut(req.info, kv.key, kv.keyLen, kv.value, kv.valueLen); + taosArrayPush(batchReq.reqs, &req); + + int32_t tlen = tSerializeSClientHbBatchReq(NULL, &batchReq); + + void* buf = (SClientHbBatchReq*)rpcMallocCont(tlen); + void* bufCopy = buf; + tSerializeSClientHbBatchReq(&bufCopy, &batchReq); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_HEARTBEAT, buf, tlen); + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + char* pRspChar = (char*)pMsg->pCont; + SClientHbBatchRsp rsp = {0}; + tDeserializeSClientHbBatchRsp(pRspChar, &rsp); + int sz = taosArrayGetSize(rsp.rsps); + ASSERT_EQ(sz, 1); + SClientHbRsp* pRsp = (SClientHbRsp*) taosArrayGet(rsp.rsps, 0); + EXPECT_EQ(pRsp->connKey.connId, 123); + EXPECT_EQ(pRsp->connKey.hbType, HEARTBEAT_TYPE_MQ); + EXPECT_EQ(pRsp->status, 0); + +#if 0 int32_t contLen = sizeof(SHeartBeatReq); SHeartBeatReq* pReq = (SHeartBeatReq*)rpcMallocCont(contLen); @@ -129,9 +161,12 @@ TEST_F(MndTestProfile, 04_HeartBeatMsg) { EXPECT_EQ(pRsp->epSet.numOfEps, 1); EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); +#endif } TEST_F(MndTestProfile, 05_KillConnMsg) { + // temporary remove since kill will use new heartbeat msg +#if 0 { int32_t contLen = sizeof(SKillConnReq); @@ -190,6 +225,7 @@ TEST_F(MndTestProfile, 05_KillConnMsg) { connId = pRsp->connId; } +#endif } TEST_F(MndTestProfile, 06_KillConnMsg_InvalidConn) { @@ -204,6 +240,8 @@ TEST_F(MndTestProfile, 06_KillConnMsg_InvalidConn) { } TEST_F(MndTestProfile, 07_KillQueryMsg) { + // temporary remove since kill will use new heartbeat msg +#if 0 { int32_t contLen = sizeof(SKillQueryReq); @@ -252,6 +290,7 @@ TEST_F(MndTestProfile, 07_KillQueryMsg) { EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } +#endif } TEST_F(MndTestProfile, 08_KillQueryMsg_InvalidConn) { diff --git a/source/dnode/vnode/src/inc/vnodeQuery.h b/source/dnode/vnode/src/inc/vnodeQuery.h index 9d40c34f009dd08c1d0f5411d971336e870d0545..51c93b5ad7a8361ea97394a90293a32a5155c322 100644 --- a/source/dnode/vnode/src/inc/vnodeQuery.h +++ b/source/dnode/vnode/src/inc/vnodeQuery.h @@ -22,6 +22,7 @@ extern "C" { #include "qworker.h" #include "vnode.h" + typedef struct SQWorkerMgmt SQHandle; int vnodeQueryOpen(SVnode *pVnode); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 2a9b4ed7141e27617f4853fce947c4eb92e64864..abdc7da1fb0961c5fcec7d1dd16a378c1a444dea 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -19,11 +19,22 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); -int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); } +int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, &pVnode->pQuery, pVnode, vnodePutReqToVQueryQ); } int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - vTrace("query message is processed"); - return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + vTrace("query message is processing"); + + switch (pMsg->msgType) { + case TDMT_VND_QUERY: + return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + case TDMT_VND_QUERY_CONTINUE: + return qWorkerProcessQueryContinueMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + case TDMT_VND_SCHEDULE_DATA_SINK: + return qWorkerProcessDataSinkMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); + default: + vError("unknown msg type:%d in query queue", pMsg->msgType); + return TSDB_CODE_VND_APP_ERROR; + } } int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index a69084f3db4966e935ebee2401db7f841781af55..a6af9ff388b7b92fa6b006cc22be01251de1e814 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -182,6 +182,12 @@ static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryE static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; + if (NULL == pDispatcher->nextOutput.pData) { + assert(pDispatcher->queryEnd); + pOutput->useconds = pDispatcher->useconds; + pOutput->precision = pDispatcher->schema.precision; + return TSDB_CODE_SUCCESS; + } SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 50f69fb567f1f2613f4b05972b20f39e9af38add..daeefba2534e3506cd4b8ed212ff045e35e05ff3 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -178,8 +178,10 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); int64_t st = 0; - *handle = pTaskInfo->dsHandle; - + if (handle) { + *handle = pTaskInfo->dsHandle; + } + while(1) { st = taosGetTimestampUs(); SSDataBlock* pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 21f57d95d5adf15daaa735f47eac7a192bb38157..bf815b26b2d3b5592796ddcaf9a5986d2dc851db 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { -// char* str = NULL; -// queryPlanToString(pLogicPlan, &str); -// printf("%s\n", str); + char* str = NULL; + queryPlanToString(pLogicPlan, &str); + printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 7883079fbed760c6f17b3a0d5d12013078920790..4030ad82adbaa0cd78ddad160403fc42ec04b420 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -23,7 +23,7 @@ extern "C" { #include "tlockfree.h" #define QWORKER_DEFAULT_SCHEDULER_NUMBER 10000 -#define QWORKER_DEFAULT_RES_CACHE_NUMBER 10000 +#define QWORKER_DEFAULT_TASK_NUMBER 10000 #define QWORKER_DEFAULT_SCH_TASK_NUMBER 10000 enum { @@ -57,7 +57,6 @@ enum { QW_ADD_ACQUIRE, }; - typedef struct SQWTaskStatus { SRWLatch lock; int32_t code; @@ -67,12 +66,15 @@ typedef struct SQWTaskStatus { bool drop; } SQWTaskStatus; -typedef struct SQWorkerTaskHandlesCache { +typedef struct SQWTaskCtx { SRWLatch lock; + int8_t sinkScheduled; + int8_t queryScheduled; + bool needRsp; qTaskInfo_t taskHandle; DataSinkHandle sinkHandle; -} SQWorkerTaskHandlesCache; +} SQWTaskCtx; typedef struct SQWSchStatus { int32_t lastAccessTs; // timestamp in second @@ -82,11 +84,15 @@ typedef struct SQWSchStatus { // Qnode/Vnode level task management typedef struct SQWorkerMgmt { - SQWorkerCfg cfg; - SRWLatch schLock; - SRWLatch resLock; - SHashObj *schHash; //key: schedulerId, value: SQWSchStatus - SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache + SQWorkerCfg cfg; + int8_t nodeType; + int32_t nodeId; + SRWLatch schLock; + SRWLatch ctxLock; + SHashObj *schHash; //key: schedulerId, value: SQWSchStatus + SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx + void *nodeObj; + putReqToQueryQFp putToQueueFp; } SQWorkerMgmt; #define QW_GOT_RES_DATA(data) (true) @@ -94,41 +100,69 @@ typedef struct SQWorkerMgmt { #define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code)) #define QW_TASK_ALREADY_EXIST(code) (TSDB_CODE_QRY_TASK_ALREADY_EXIST == (code)) -#define QW_TASK_READY_RESP(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || status == JOB_TASK_STATUS_PARTIAL_SUCCEED) -#define QW_SET_QTID(id, qid, tid) do { *(uint64_t *)(id) = (qid); *(uint64_t *)((char *)(id) + sizeof(qid)) = (tid); } while (0) -#define QW_GET_QTID(id, qid, tid) do { (qid) = *(uint64_t *)(id); (tid) = *(uint64_t *)((char *)(id) + sizeof(qid)); } while (0) - +#define QW_TASK_READY(status) (status == JOB_TASK_STATUS_SUCCEED || status == JOB_TASK_STATUS_FAILED || status == JOB_TASK_STATUS_CANCELLED || status == JOB_TASK_STATUS_PARTIAL_SUCCEED) +#define QW_SET_QTID(id, qId, tId) do { *(uint64_t *)(id) = (qId); *(uint64_t *)((char *)(id) + sizeof(qId)) = (tId); } while (0) +#define QW_GET_QTID(id, qId, tId) do { (qId) = *(uint64_t *)(id); (tId) = *(uint64_t *)((char *)(id) + sizeof(qId)); } while (0) +#define QW_IDS() sId, qId, tId #define QW_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define QW_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) -#define QW_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); terrno = _code; return _code; } } while (0) #define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) +#define QW_ELOG(param, ...) qError("QW:%p " param, mgmt, __VA_ARGS__) +#define QW_DLOG(param, ...) qDebug("QW:%p " param, mgmt, __VA_ARGS__) + +#define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) +#define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) + +#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) + +#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) + + +#define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 + #define QW_LOCK(type, _lock) do { \ if (QW_READ == (type)) { \ - if ((*(_lock)) < 0) assert(0); \ - taosRLockLatch(_lock); \ - qDebug("QW RLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ + qDebug("QW RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRLockLatch(_lock); \ + qDebug("QW RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) > 0); \ } else { \ - if ((*(_lock)) < 0) assert(0); \ + assert(atomic_load_32((_lock)) >= 0); \ + qDebug("QW WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWLockLatch(_lock); \ - qDebug("QW WLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + qDebug("QW WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ } \ } while (0) - + #define QW_UNLOCK(type, _lock) do { \ if (QW_READ == (type)) { \ - if ((*(_lock)) <= 0) assert(0); \ + assert(atomic_load_32((_lock)) > 0); \ + qDebug("QW RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosRUnLockLatch(_lock); \ - qDebug("QW RULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + qDebug("QW RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } else { \ - if ((*(_lock)) <= 0) assert(0); \ + assert(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY); \ + qDebug("QW WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ taosWUnLockLatch(_lock); \ - qDebug("QW WULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \ + qDebug("QW WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + assert(atomic_load_32((_lock)) >= 0); \ } \ } while (0) -static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt); + + +int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch); +int32_t qwAcquireAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch); +int32_t qwAcquireTask(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task); #ifdef __cplusplus diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 2a395fcfe10962fdd48b34fceb3c6d7490650597..1be190065a093b55498727956140a4982f172ccb 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -8,7 +8,7 @@ #include "tname.h" #include "dataSinkMgt.h" -int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { +int32_t qwValidateStatus(SQWorkerMgmt *mgmt, int8_t oriStatus, int8_t newStatus, uint64_t sId, uint64_t qId, uint64_t tId) { int32_t code = 0; if (oriStatus == newStatus) { @@ -62,7 +62,7 @@ int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { break; default: - qError("invalid task status:%d", oriStatus); + QW_TASK_ELOG("invalid task status:%d", oriStatus); return TSDB_CODE_QRY_APP_ERROR; } @@ -70,22 +70,27 @@ int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { _return: - qError("invalid task status, from %d to %d", oriStatus, newStatus); - QW_ERR_RET(code); + QW_TASK_ELOG("invalid task status update from %d to %d", oriStatus, newStatus); + QW_RET(code); } -int32_t qwUpdateTaskInfo(SQWTaskStatus *task, int8_t type, void *data) { +int32_t qwUpdateTaskInfo(SQWorkerMgmt *mgmt, SQWTaskStatus *task, int8_t type, void *data, uint64_t sId, uint64_t qId, uint64_t tId) { int32_t code = 0; + int8_t origStatus = 0; switch (type) { case QW_TASK_INFO_STATUS: { int8_t newStatus = *(int8_t *)data; - QW_ERR_RET(qwValidateStatus(task->status, newStatus)); + QW_ERR_RET(qwValidateStatus(mgmt, task->status, newStatus, QW_IDS())); + + origStatus = task->status; task->status = newStatus; + + QW_TASK_DLOG("task status updated from %d to %d", origStatus, newStatus); break; } default: - qError("uknown task info type:%d", type); + QW_TASK_ELOG("unknown task info, type:%d", type); return TSDB_CODE_QRY_APP_ERROR; } @@ -96,27 +101,27 @@ int32_t qwAddTaskHandlesToCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId, char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); - SQWorkerTaskHandlesCache resCache = {0}; + SQWTaskCtx resCache = {0}; resCache.taskHandle = taskHandle; resCache.sinkHandle = sinkHandle; - QW_LOCK(QW_WRITE, &mgmt->resLock); - if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerTaskHandlesCache))) { - QW_UNLOCK(QW_WRITE, &mgmt->resLock); - qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to resHash failed", qId, tId); + QW_LOCK(QW_WRITE, &mgmt->ctxLock); + if (0 != taosHashPut(mgmt->ctxHash, id, sizeof(id), &resCache, sizeof(SQWTaskCtx))) { + QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + QW_TASK_ELOG("taosHashPut task ctx to ctxHash failed, taskHandle:%p, sinkHandle:%p", taskHandle, sinkHandle); return TSDB_CODE_QRY_APP_ERROR; } - QW_UNLOCK(QW_WRITE, &mgmt->resLock); + QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); return TSDB_CODE_SUCCESS; } -static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { +int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { SQWSchStatus newSch = {0}; newSch.tasksHash = taosHashInit(mgmt->cfg.maxSchTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == newSch.tasksHash) { - qError("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); + QW_SCH_DLOG("taosHashInit %d failed", mgmt->cfg.maxSchTaskNum); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -126,14 +131,18 @@ static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, if (0 != code) { if (!HASH_NODE_EXIST(code)) { QW_UNLOCK(QW_WRITE, &mgmt->schLock); - qError("taosHashPut sId[%"PRIx64"] to scheduleHash failed", sId); + QW_SCH_ELOG("taosHashPut new sch to scheduleHash failed, errno:%d", errno); taosHashCleanup(newSch.tasksHash); return TSDB_CODE_QRY_APP_ERROR; } } QW_UNLOCK(QW_WRITE, &mgmt->schLock); - if (TSDB_CODE_SUCCESS == qwAcquireScheduler(rwType, mgmt, sId, sch, QW_NOT_EXIST_ADD)) { + if (TSDB_CODE_SUCCESS == qwAcquireScheduler(rwType, mgmt, sId, sch)) { + if (code) { + taosHashCleanup(newSch.tasksHash); + } + return TSDB_CODE_SUCCESS; } } @@ -141,7 +150,7 @@ static int32_t qwAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, return TSDB_CODE_SUCCESS; } -static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt) { +int32_t qwAcquireSchedulerImpl(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt) { QW_LOCK(rwType, &mgmt->schLock); *sch = taosHashGet(mgmt->schHash, &sId, sizeof(sId)); if (NULL == (*sch)) { @@ -159,113 +168,119 @@ static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t s return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { +int32_t qwAcquireAddScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { + return qwAcquireSchedulerImpl(rwType, mgmt, sId, sch, QW_NOT_EXIST_ADD); +} + +int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch) { + return qwAcquireSchedulerImpl(rwType, mgmt, sId, sch, QW_NOT_EXIST_RET_ERR); +} + +void qwReleaseScheduler(int32_t rwType, SQWorkerMgmt *mgmt) { QW_UNLOCK(rwType, &mgmt->schLock); } -static int32_t qwAcquireTaskImpl(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task) { +int32_t qwAddTaskImpl(SQWorkerMgmt *mgmt, SQWSchStatus *sch, int32_t rwType, uint64_t qId, uint64_t tId, int32_t status, int32_t eOpt, SQWTaskStatus **task) { + int32_t code = 0; + char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); - QW_LOCK(rwType, &sch->tasksLock); - *task = taosHashGet(sch->tasksHash, id, sizeof(id)); - if (NULL == (*task)) { - QW_UNLOCK(rwType, &sch->tasksLock); + SQWTaskStatus ntask = {0}; + ntask.status = status; + + QW_LOCK(QW_WRITE, &sch->tasksLock); + code = taosHashPut(sch->tasksHash, id, sizeof(id), &ntask, sizeof(ntask)); + if (0 != code) { + QW_UNLOCK(QW_WRITE, &sch->tasksLock); + if (HASH_NODE_EXIST(code)) { + if (QW_EXIST_ACQUIRE == eOpt && rwType && task) { + QW_ERR_RET(qwAcquireTask(mgmt, rwType, sch, qId, tId, task)); + } else if (QW_EXIST_RET_ERR == eOpt) { + return TSDB_CODE_QRY_TASK_ALREADY_EXIST; + } else { + assert(0); + } + } else { + qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", qId, tId); + return TSDB_CODE_QRY_APP_ERROR; + } + } + + QW_UNLOCK(QW_WRITE, &sch->tasksLock); - return TSDB_CODE_QRY_TASK_NOT_EXIST; + if (QW_EXIST_ACQUIRE == eOpt && rwType && task) { + QW_ERR_RET(qwAcquireTask(mgmt, rwType, sch, qId, tId, task)); } return TSDB_CODE_SUCCESS; } -static int32_t qwAcquireTask(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task) { - return qwAcquireTaskImpl(rwType, sch, qId, tId, task); -} +int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status) { + SQWSchStatus *tsch = NULL; + int32_t code = 0; + QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &tsch)); -static FORCE_INLINE void qwReleaseTask(int32_t rwType, SQWSchStatus *sch) { - QW_UNLOCK(rwType, &sch->tasksLock); + QW_ERR_JRET(qwAddTaskImpl(mgmt, tsch, 0, qId, tId, status, QW_EXIST_RET_ERR, NULL)); + +_return: + + qwReleaseScheduler(QW_READ, mgmt); + QW_ERR_RET(code); } -int32_t qwAddTaskToSch(int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int8_t status, int32_t eOpt, SQWTaskStatus **task) { - int32_t code = 0; +int32_t qwAcquireTaskImpl(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int32_t status, int32_t nOpt, SQWTaskStatus **task) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); - SQWTaskStatus ntask = {0}; - ntask.status = status; - - while (true) { - QW_LOCK(QW_WRITE, &sch->tasksLock); - int32_t code = taosHashPut(sch->tasksHash, id, sizeof(id), &ntask, sizeof(ntask)); - if (0 != code) { - QW_UNLOCK(QW_WRITE, &sch->tasksLock); - if (HASH_NODE_EXIST(code)) { - if (QW_EXIST_ACQUIRE == eOpt && rwType && task) { - if (qwAcquireTask(rwType, sch, qId, tId, task)) { - continue; - } - } else if (QW_EXIST_RET_ERR == eOpt) { - return TSDB_CODE_QRY_TASK_ALREADY_EXIST; - } else { - assert(0); - } - - break; - } else { - qError("taosHashPut queryId[%"PRIx64"] taskId[%"PRIx64"] to scheduleHash failed", qId, tId); - return TSDB_CODE_QRY_APP_ERROR; - } - } - - QW_UNLOCK(QW_WRITE, &sch->tasksLock); + QW_LOCK(rwType, &sch->tasksLock); + *task = taosHashGet(sch->tasksHash, id, sizeof(id)); + if (NULL == (*task)) { + QW_UNLOCK(rwType, &sch->tasksLock); - if (rwType && task) { - if (TSDB_CODE_SUCCESS == qwAcquireTask(rwType, sch, qId, tId, task)) { - return TSDB_CODE_SUCCESS; - } + if (QW_NOT_EXIST_ADD == nOpt) { + QW_ERR_RET(qwAddTaskImpl(mgmt, sch, rwType, qId, tId, status, QW_EXIST_ACQUIRE, task)); + } else if (QW_NOT_EXIST_RET_ERR == nOpt) { + return TSDB_CODE_QRY_TASK_NOT_EXIST; } else { - break; + assert(0); } - } + } return TSDB_CODE_SUCCESS; } -static int32_t qwAddTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int32_t status, int32_t eOpt, SQWSchStatus **sch, SQWTaskStatus **task) { - SQWSchStatus *tsch = NULL; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &tsch, QW_NOT_EXIST_ADD)); +int32_t qwAcquireTask(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, SQWTaskStatus **task) { + return qwAcquireTaskImpl(mgmt, rwType, sch, qId, tId, 0, QW_NOT_EXIST_RET_ERR, task); +} - int32_t code = qwAddTaskToSch(QW_READ, tsch, qId, tId, status, eOpt, task); - if (code) { - qwReleaseScheduler(QW_WRITE, mgmt); - } +int32_t qwAcquireAddTask(SQWorkerMgmt *mgmt, int32_t rwType, SQWSchStatus *sch, uint64_t qId, uint64_t tId, int32_t status, SQWTaskStatus **task) { + return qwAcquireTaskImpl(mgmt, rwType, sch, qId, tId, status, QW_NOT_EXIST_ADD, task); +} - if (NULL == task) { - qwReleaseScheduler(QW_READ, mgmt); - } else if (sch) { - *sch = tsch; - } - QW_RET(code); +void qwReleaseTask(int32_t rwType, SQWSchStatus *sch) { + QW_UNLOCK(rwType, &sch->tasksLock); } -static FORCE_INLINE int32_t qwAcquireTaskHandles(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWorkerTaskHandlesCache **handles) { + +int32_t qwAcquireTaskCtx(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t queryId, uint64_t taskId, SQWTaskCtx **handles) { char id[sizeof(queryId) + sizeof(taskId)] = {0}; QW_SET_QTID(id, queryId, taskId); - QW_LOCK(rwType, &mgmt->resLock); - *handles = taosHashGet(mgmt->resHash, id, sizeof(id)); + QW_LOCK(rwType, &mgmt->ctxLock); + *handles = taosHashGet(mgmt->ctxHash, id, sizeof(id)); if (NULL == (*handles)) { - QW_UNLOCK(rwType, &mgmt->resLock); + QW_UNLOCK(rwType, &mgmt->ctxLock); return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST; } return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void qwReleaseTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt) { - QW_UNLOCK(rwType, &mgmt->resLock); +void qwReleaseTaskResCache(int32_t rwType, SQWorkerMgmt *mgmt) { + QW_UNLOCK(rwType, &mgmt->ctxLock); } @@ -273,7 +288,7 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRs SQWSchStatus *sch = NULL; int32_t taskNum = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); sch->lastAccessTs = taosGetTimestampSec(); @@ -319,7 +334,7 @@ int32_t qwGetSchTasksStatus(SQWorkerMgmt *mgmt, uint64_t sId, SSchedulerStatusRs int32_t qwUpdateSchLastAccess(SQWorkerMgmt *mgmt, uint64_t sId) { SQWSchStatus *sch = NULL; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); sch->lastAccessTs = taosGetTimestampSec(); @@ -333,12 +348,12 @@ int32_t qwUpdateTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); - QW_ERR_JRET(qwAcquireTask(QW_READ, sch, qId, tId, &task)); + QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task)); QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); + qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()); QW_UNLOCK(QW_WRITE, &task->lock); _return: @@ -355,12 +370,12 @@ int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint SQWTaskStatus *task = NULL; int32_t code = 0; - if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) { + if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch)) { *taskStatus = JOB_TASK_STATUS_NULL; return TSDB_CODE_SUCCESS; } - if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { + if (qwAcquireTask(mgmt, QW_READ, sch, queryId, taskId, &task)) { qwReleaseScheduler(QW_READ, mgmt); *taskStatus = JOB_TASK_STATUS_NULL; @@ -376,22 +391,15 @@ int32_t qwGetTaskStatus(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint } -int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { +int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD)); + QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &sch)); + + QW_ERR_JRET(qwAcquireAddTask(mgmt, QW_READ, sch, qId, tId, JOB_TASK_STATUS_NOT_START, &task)); - if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { - qwReleaseScheduler(QW_READ, mgmt); - - code = qwAddTask(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task); - if (code) { - qwReleaseScheduler(QW_READ, mgmt); - QW_ERR_RET(code); - } - } QW_LOCK(QW_WRITE, &task->lock); @@ -409,10 +417,10 @@ int32_t qwCancelTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_ return TSDB_CODE_SUCCESS; } else if (task->status == JOB_TASK_STATUS_FAILED || task->status == JOB_TASK_STATUS_SUCCEED || task->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { newStatus = JOB_TASK_STATUS_CANCELLED; - QW_ERR_JRET(qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus)); + QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS())); } else { newStatus = JOB_TASK_STATUS_CANCELLING; - QW_ERR_JRET(qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus)); + QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS())); } QW_UNLOCK(QW_WRITE, &task->lock); @@ -441,55 +449,87 @@ _return: QW_RET(code); } -int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { + +// caller should make sure task is not running +int32_t qwDropTaskCtx(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { + char id[sizeof(qId) + sizeof(tId)] = {0}; + QW_SET_QTID(id, qId, tId); + + QW_LOCK(QW_WRITE, &mgmt->ctxLock); + SQWTaskCtx *ctx = taosHashGet(mgmt->ctxHash, id, sizeof(id)); + if (NULL == ctx) { + QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST; + } + + if (ctx->taskHandle) { + qDestroyTask(ctx->taskHandle); + ctx->taskHandle = NULL; + } + + if (ctx->sinkHandle) { + dsDestroyDataSinker(ctx->sinkHandle); + ctx->sinkHandle = NULL; + } + + if (taosHashRemove(mgmt->ctxHash, id, sizeof(id))) { + QW_TASK_ELOG("taosHashRemove from ctx hash failed, id:%s", id); + + QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + return TSDB_CODE_QRY_RES_CACHE_NOT_EXIST; + } + + QW_UNLOCK(QW_WRITE, &mgmt->ctxLock); + + return TSDB_CODE_SUCCESS; +} + + +int32_t qwDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; - char id[sizeof(queryId) + sizeof(taskId)] = {0}; - QW_SET_QTID(id, queryId, taskId); + + char id[sizeof(qId) + sizeof(tId)] = {0}; + QW_SET_QTID(id, qId, tId); - QW_LOCK(QW_WRITE, &mgmt->resLock); - if (mgmt->resHash) { - taosHashRemove(mgmt->resHash, id, sizeof(id)); - } - QW_UNLOCK(QW_WRITE, &mgmt->resLock); + qwDropTaskCtx(mgmt, sId, qId, tId); - if (TSDB_CODE_SUCCESS != qwAcquireScheduler(QW_WRITE, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) { - qWarn("scheduler %"PRIx64" doesn't exist", sId); + if (qwAcquireScheduler(QW_WRITE, mgmt, sId, &sch)) { + QW_TASK_WLOG("scheduler does not exist, sch:%p", sch); return TSDB_CODE_SUCCESS; } - if (qwAcquireTask(QW_WRITE, sch, queryId, taskId, &task)) { + if (qwAcquireTask(mgmt, QW_WRITE, sch, qId, tId, &task)) { qwReleaseScheduler(QW_WRITE, mgmt); - qWarn("scheduler %"PRIx64" queryId %"PRIx64" taskId:%"PRIx64" doesn't exist", sId, queryId, taskId); + QW_TASK_WLOG("task does not exist, task:%p", task); return TSDB_CODE_SUCCESS; } - taosHashRemove(sch->tasksHash, id, sizeof(id)); + QW_TASK_DLOG("drop task, status:%d, code:%x, ready:%d, cancel:%d, drop:%d", task->status, task->code, task->ready, task->cancel, task->drop); + + if (taosHashRemove(sch->tasksHash, id, sizeof(id))) { + QW_TASK_ELOG("taosHashRemove task from hash failed, task:%p", task); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + +_return: qwReleaseTask(QW_WRITE, sch); qwReleaseScheduler(QW_WRITE, mgmt); - return TSDB_CODE_SUCCESS; + QW_RET(code); } -int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId) { +int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD)); + QW_ERR_RET(qwAcquireAddScheduler(QW_READ, mgmt, sId, &sch)); - if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { - qwReleaseScheduler(QW_READ, mgmt); - - code = qwAddTask(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_NOT_START, QW_EXIST_ACQUIRE, &sch, &task); - if (code) { - qwReleaseScheduler(QW_READ, mgmt); - QW_ERR_RET(code); - } - } + QW_ERR_JRET(qwAcquireAddTask(mgmt, QW_READ, sch, qId, tId, JOB_TASK_STATUS_NOT_START, &task)); QW_LOCK(QW_WRITE, &task->lock); @@ -500,7 +540,7 @@ int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uin if (task->status == JOB_TASK_STATUS_EXECUTING) { newStatus = JOB_TASK_STATUS_DROPPING; - QW_ERR_JRET(qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus)); + QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS())); } else if (task->status == JOB_TASK_STATUS_CANCELLING || task->status == JOB_TASK_STATUS_DROPPING || task->status == JOB_TASK_STATUS_NOT_START) { QW_UNLOCK(QW_WRITE, &task->lock); qwReleaseTask(QW_READ, sch); @@ -512,11 +552,12 @@ int32_t qwCancelDropTask(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uin qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - QW_ERR_RET(qwDropTask(mgmt, sId, queryId, taskId)); + QW_ERR_RET(qwDropTask(mgmt, sId, qId, tId)); return TSDB_CODE_SUCCESS; } QW_UNLOCK(QW_WRITE, &task->lock); + qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); @@ -738,30 +779,35 @@ int32_t qwBuildAndSendShowFetchRsp(SRpcMsg *pMsg, SVShowTablesFetchReq* pFetchRe return TSDB_CODE_SUCCESS; } -int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg, int32_t rspCode) { +int32_t qwCheckAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); - QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); + QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task)); QW_LOCK(QW_WRITE, &task->lock); if (QW_READY_NOT_RECEIVED == task->ready) { + QW_SCH_TASK_DLOG("ready not received, ready:%d", task->ready); + goto _return; + } else if (QW_READY_RECEIVED == task->ready) { + task->ready = QW_READY_RESPONSED; + int32_t rspCode = task->code; + QW_UNLOCK(QW_WRITE, &task->lock); - qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - return TSDB_CODE_SUCCESS; - } else if (QW_READY_RECEIVED == task->ready) { - QW_ERR_JRET(qwBuildAndSendReadyRsp(pMsg, rspCode)); + QW_ERR_RET(qwBuildAndSendReadyRsp(pMsg, rspCode)); + + QW_SCH_TASK_DLOG("ready response sent, ready:%d", task->ready); - task->ready = QW_READY_RESPONSED; + return TSDB_CODE_SUCCESS; } else if (QW_READY_RESPONSED == task->ready) { - qError("query response already send"); + QW_SCH_TASK_ELOG("ready response already send, ready:%d", task->ready); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } else { assert(0); @@ -772,7 +818,6 @@ _return: if (task) { QW_UNLOCK(QW_WRITE, &task->lock); qwReleaseTask(QW_READ, sch); - } qwReleaseScheduler(QW_READ, mgmt); @@ -780,34 +825,39 @@ _return: QW_RET(code); } -int32_t qwSetAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { +int32_t qwSetAndSendReadyRsp(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); + QW_ERR_RET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); - QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); + QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task)); QW_LOCK(QW_WRITE, &task->lock); - if (QW_TASK_READY_RESP(task->status)) { - QW_ERR_JRET(qwBuildAndSendReadyRsp(pMsg, task->code)); + int8_t status = task->status; + int32_t errCode = task->code; + + if (QW_TASK_READY(status)) { task->ready = QW_READY_RESPONSED; + + QW_UNLOCK(QW_WRITE, &task->lock); + + QW_ERR_JRET(qwBuildAndSendReadyRsp(pMsg, errCode)); + + QW_SCH_TASK_DLOG("task ready responsed, status:%d", status); } else { task->ready = QW_READY_RECEIVED; + QW_UNLOCK(QW_WRITE, &task->lock); - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); - - return TSDB_CODE_SUCCESS; + QW_SCH_TASK_DLOG("task ready NOT responsed, status:%d", status); } _return: if (task) { - QW_UNLOCK(QW_WRITE, &task->lock); qwReleaseTask(QW_READ, sch); } @@ -816,7 +866,7 @@ _return: QW_RET(code); } -int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, bool *needStop) { +int32_t qwCheckAndProcessTaskDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, bool *needStop) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -824,44 +874,39 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI *needStop = false; - if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)) { + if (qwAcquireScheduler(QW_READ, mgmt, sId, &sch)) { return TSDB_CODE_SUCCESS; } - if (qwAcquireTask(QW_READ, sch, queryId, taskId, &task)) { + if (qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task)) { qwReleaseScheduler(QW_READ, mgmt); return TSDB_CODE_SUCCESS; } - - QW_LOCK(QW_READ, &task->lock); - if ((!task->cancel) && (!task->drop)) { - qError("no cancel or drop, but task:%"PRIx64" exists", taskId); - - QW_UNLOCK(QW_READ, &task->lock); - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); - - QW_RET(TSDB_CODE_QRY_APP_ERROR); + if ((!atomic_load_8(&task->cancel)) && (!atomic_load_8(&task->drop))) { + QW_TASK_ELOG("no cancel or drop but task exists, status:%d", atomic_load_8(&task->status)); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - QW_UNLOCK(QW_READ, &task->lock); - *needStop = true; - if (task->cancel) { + if (atomic_load_8(&task->cancel)) { QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); + code = qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()); QW_UNLOCK(QW_WRITE, &task->lock); + + QW_ERR_JRET(code); } if (task->drop) { qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); - return qwDropTask(mgmt, sId, queryId, taskId); + QW_RET(qwDropTask(mgmt, sId, qId, tId)); } +_return: + qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); @@ -875,31 +920,27 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 int32_t code = 0; int8_t newStatus = JOB_TASK_STATUS_CANCELLED; - code = qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD); + code = qwAcquireAddScheduler(QW_READ, mgmt, sId, &sch); if (code) { - qError("sId:%"PRIx64" not in cache", sId); + QW_TASK_ELOG("sId:%"PRIx64" not in cache", sId); QW_ERR_RET(code); } - code = qwAcquireTask(QW_READ, sch, qId, tId, &task); + code = qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task); if (code) { - qwReleaseScheduler(QW_READ, mgmt); - - if (JOB_TASK_STATUS_PARTIAL_SUCCEED == status || JOB_TASK_STATUS_SUCCEED == status) { - qError("sId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", sId, qId, tId); - QW_ERR_RET(code); - } - - QW_ERR_RET(qwAddTask(mgmt, sId, qId, tId, status, QW_EXIST_ACQUIRE, &sch, &task)); + QW_TASK_ELOG("sId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", sId, qId, tId); + QW_ERR_RET(code); } + QW_LOCK(QW_WRITE, &task->lock); + if (task->cancel) { - QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus); - QW_UNLOCK(QW_WRITE, &task->lock); + qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &newStatus, QW_IDS()); } if (task->drop) { + QW_UNLOCK(QW_WRITE, &task->lock); + qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); @@ -909,11 +950,11 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 } if (!(task->cancel || task->drop)) { - QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); + qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS()); task->code = errCode; - QW_UNLOCK(QW_WRITE, &task->lock); } + + QW_UNLOCK(QW_WRITE, &task->lock); qwReleaseTask(QW_READ, sch); qwReleaseScheduler(QW_READ, mgmt); @@ -921,8 +962,89 @@ int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint6 return TSDB_CODE_SUCCESS; } +int32_t qwScheduleDataSink(SQWTaskCtx *handles, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { + if (atomic_load_8(&handles->sinkScheduled)) { + qDebug("data sink already scheduled"); + return TSDB_CODE_SUCCESS; + } + + SSinkDataReq * req = (SSinkDataReq *)rpcMallocCont(sizeof(SSinkDataReq)); + if (NULL == req) { + qError("rpcMallocCont %d failed", (int32_t)sizeof(SSinkDataReq)); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + req->header.vgId = mgmt->nodeId; + req->sId = sId; + req->queryId = queryId; + req->taskId = taskId; + + SRpcMsg pNewMsg = { + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .msgType = TDMT_VND_SCHEDULE_DATA_SINK, + .pCont = req, + .contLen = sizeof(SSinkDataReq), + .code = 0, + }; + + int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg); + if (TSDB_CODE_SUCCESS != code) { + qError("put data sink schedule msg to queue failed, code:%x", code); + rpcFreeCont(req); + QW_ERR_RET(code); + } + + qDebug("put data sink schedule msg to query queue"); -int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { + return TSDB_CODE_SUCCESS; +} + +int32_t qwScheduleQuery(SQWTaskCtx *handles, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) { + if (atomic_load_8(&handles->queryScheduled)) { + QW_SCH_TASK_ELOG("query already scheduled, queryScheduled:%d", handles->queryScheduled); + return TSDB_CODE_SUCCESS; + } + + QW_ERR_RET(qwUpdateTaskStatus(mgmt, sId, qId, tId, JOB_TASK_STATUS_EXECUTING)); + + SQueryContinueReq * req = (SQueryContinueReq *)rpcMallocCont(sizeof(SQueryContinueReq)); + if (NULL == req) { + QW_SCH_TASK_ELOG("rpcMallocCont %d failed", (int32_t)sizeof(SQueryContinueReq)); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + req->header.vgId = mgmt->nodeId; + req->sId = sId; + req->queryId = qId; + req->taskId = tId; + + SRpcMsg pNewMsg = { + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .msgType = TDMT_VND_QUERY_CONTINUE, + .pCont = req, + .contLen = sizeof(SQueryContinueReq), + .code = 0, + }; + + int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg); + if (TSDB_CODE_SUCCESS != code) { + QW_SCH_TASK_ELOG("put query continue msg to queue failed, code:%x", code); + rpcFreeCont(req); + QW_ERR_RET(code); + } + + handles->queryScheduled = true; + + QW_SCH_TASK_DLOG("put query continue msg to query queue, vgId:%d", mgmt->nodeId); + + return TSDB_CODE_SUCCESS; +} + + + +int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, SRpcMsg *pMsg) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -932,22 +1054,30 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 int32_t dataLength = 0; SRetrieveTableRsp *rsp = NULL; bool queryEnd = false; - SQWorkerTaskHandlesCache *handles = NULL; + SQWTaskCtx *handles = NULL; + int8_t status = 0; - QW_ERR_JRET(qwAcquireTaskHandles(QW_READ, mgmt, queryId, taskId, &handles)); + QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, mgmt, qId, tId, &handles)); + QW_LOCK(QW_WRITE, &handles->lock); + + if (handles->needRsp) { + QW_UNLOCK(QW_WRITE, &handles->lock); + QW_SCH_TASK_ELOG("last fetch not responsed, needRsp:%d", handles->needRsp); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } - QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); - QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); + QW_UNLOCK(QW_WRITE, &handles->lock); - QW_LOCK(QW_READ, &task->lock); + QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch)); + QW_ERR_JRET(qwAcquireTask(mgmt, QW_READ, sch, qId, tId, &task)); if (task->cancel || task->drop) { - qError("task is already cancelled or dropped"); + QW_SCH_TASK_ELOG("task is already cancelled or dropped, cancel:%d, drop:%d", task->cancel, task->drop); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } if (task->status != JOB_TASK_STATUS_EXECUTING && task->status != JOB_TASK_STATUS_PARTIAL_SUCCEED) { - qError("invalid status %d for fetch", task->status); + QW_SCH_TASK_ELOG("invalid status %d for fetch", task->status); QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -955,6 +1085,9 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 if (dataLength > 0) { SOutputData output = {0}; + + QW_SCH_TASK_DLOG("task got data in sink, dataLength:%d", dataLength); + QW_ERR_JRET(qwInitFetchRsp(dataLength, &rsp)); output.pData = rsp->data; @@ -974,29 +1107,39 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 if (DS_BUF_EMPTY == output.bufStatus && output.queryEnd) { rsp->completed = 1; + + status = JOB_TASK_STATUS_SUCCEED; + + QW_SCH_TASK_DLOG("task all fetched, status:%d", status); + QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS())); } + // Note: schedule data sink firstly and will schedule query after it's done if (output.needSchedule) { - //TODO - } - - if ((!output.queryEnd) && DS_BUF_LOW == output.bufStatus) { - //TODO - //UPDATE STATUS TO EXECUTING + QW_SCH_TASK_DLOG("sink need schedule, queryEnd:%d", output.queryEnd); + QW_ERR_JRET(qwScheduleDataSink(handles, mgmt, sId, qId, tId, pMsg)); + } else if ((!output.queryEnd) && (DS_BUF_LOW == output.bufStatus || DS_BUF_EMPTY == output.bufStatus)) { + QW_SCH_TASK_DLOG("task not end, need to continue, bufStatus:%d", output.bufStatus); + QW_ERR_JRET(qwScheduleQuery(handles, mgmt, sId, qId, tId, pMsg)); } } else { if (dataLength < 0) { - qError("invalid length from dsGetDataLength, length:%d", dataLength); + QW_SCH_TASK_ELOG("invalid length from dsGetDataLength, length:%d", dataLength); QW_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } if (queryEnd) { - QW_ERR_JRET(qwQueryPostProcess(mgmt, sId, queryId, taskId, JOB_TASK_STATUS_SUCCEED, code)); + status = JOB_TASK_STATUS_SUCCEED; + + QW_SCH_TASK_DLOG("no data in sink and query end, dataLength:%d", dataLength); + + QW_ERR_JRET(qwUpdateTaskInfo(mgmt, task, QW_TASK_INFO_STATUS, &status, QW_IDS())); } else { - if (task->status != JOB_TASK_STATUS_EXECUTING) { - qError("invalid status %d for fetch without res", task->status); - QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } + assert(0 == handles->needRsp); + + // MUST IN SCHEDULE OR IN SINK SCHEDULE + + QW_SCH_TASK_DLOG("no res data in sink, need response later, queryEnd:%d", queryEnd); QW_LOCK(QW_WRITE, &handles->lock); handles->needRsp = true; @@ -1009,7 +1152,6 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 _return: if (task) { - QW_UNLOCK(QW_READ, &task->lock); qwReleaseTask(QW_READ, sch); } @@ -1028,7 +1170,12 @@ _return: QW_RET(code); } -int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, putReqToQueryQFp fp) { + if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp) { + qError("invalid param to init qworker"); + QW_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + SQWorkerMgmt *mgmt = calloc(1, sizeof(SQWorkerMgmt)); if (NULL == mgmt) { qError("calloc %d failed", (int32_t)sizeof(SQWorkerMgmt)); @@ -1037,29 +1184,46 @@ int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { if (cfg) { mgmt->cfg = *cfg; + if (0 == mgmt->cfg.maxSchedulerNum) { + mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER; + } + if (0 == mgmt->cfg.maxTaskNum) { + mgmt->cfg.maxTaskNum = QWORKER_DEFAULT_TASK_NUMBER; + } + if (0 == mgmt->cfg.maxSchTaskNum) { + mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER; + } } else { mgmt->cfg.maxSchedulerNum = QWORKER_DEFAULT_SCHEDULER_NUMBER; - mgmt->cfg.maxResCacheNum = QWORKER_DEFAULT_RES_CACHE_NUMBER; + mgmt->cfg.maxTaskNum = QWORKER_DEFAULT_TASK_NUMBER; mgmt->cfg.maxSchTaskNum = QWORKER_DEFAULT_SCH_TASK_NUMBER; } mgmt->schHash = taosHashInit(mgmt->cfg.maxSchedulerNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); if (NULL == mgmt->schHash) { tfree(mgmt); - QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler hash failed", mgmt->cfg.maxSchedulerNum); + qError("init %d scheduler hash failed", mgmt->cfg.maxSchedulerNum); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - mgmt->resHash = taosHashInit(mgmt->cfg.maxResCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (NULL == mgmt->resHash) { + mgmt->ctxHash = taosHashInit(mgmt->cfg.maxTaskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == mgmt->ctxHash) { taosHashCleanup(mgmt->schHash); mgmt->schHash = NULL; tfree(mgmt); - - QW_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d res cache hash failed", mgmt->cfg.maxResCacheNum); + qError("init %d task ctx hash failed", mgmt->cfg.maxTaskNum); + QW_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } + mgmt->nodeType = nodeType; + mgmt->nodeId = nodeId; + mgmt->nodeObj = nodeObj; + mgmt->putToQueueFp = fp; + *qWorkerMgmt = mgmt; + qDebug("qworker initialized for node, type:%d, id:%d, handle:%p", mgmt->nodeType, mgmt->nodeId, mgmt); + return TSDB_CODE_SUCCESS; } @@ -1069,105 +1233,164 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { } int32_t code = 0; + bool queryRsped = false; + bool needStop = false; + struct SSubplan *plan = NULL; SSubQueryMsg *msg = pMsg->pCont; + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; + int32_t rspCode = 0; + if (NULL == msg || pMsg->contLen <= sizeof(*msg)) { - qError("invalid query msg"); + QW_ELOG("invalid query msg, contLen:%d", pMsg->contLen); QW_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->sId = htobe64(msg->sId); - msg->queryId = htobe64(msg->queryId); - msg->taskId = htobe64(msg->taskId); + msg->sId = be64toh(msg->sId); + msg->queryId = be64toh(msg->queryId); + msg->taskId = be64toh(msg->taskId); msg->contentLen = ntohl(msg->contentLen); - bool queryRsped = false; - bool needStop = false; - struct SSubplan *plan = NULL; + uint64_t sId = msg->sId; + uint64_t qId = msg->queryId; + uint64_t tId = msg->taskId; - QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop)); + QW_ERR_JRET(qwCheckAndProcessTaskDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop)); if (needStop) { - qWarn("task need stop"); - QW_ERR_JRET(TSDB_CODE_QRY_TASK_CANCELLED); + QW_TASK_DLOG("task need stop, msgLen:%d", msg->contentLen); + qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_QRY_TASK_CANCELLED); + QW_ERR_RET(TSDB_CODE_QRY_TASK_CANCELLED); } + + QW_ERR_JRET(qwAddTask(qWorkerMgmt, sId, qId, tId, JOB_TASK_STATUS_EXECUTING)); code = qStringToSubplan(msg->msg, &plan); if (TSDB_CODE_SUCCESS != code) { - qError("schId:%"PRIx64",qId:%"PRIx64",taskId:%"PRIx64" string to subplan failed, code:%d", msg->sId, msg->queryId, msg->taskId, code); + QW_TASK_ELOG("string to subplan failed, code:%d", code); QW_ERR_JRET(code); } qTaskInfo_t pTaskInfo = NULL; code = qCreateExecTask(node, 0, (struct SSubplan *)plan, &pTaskInfo); if (code) { - qError("qCreateExecTask failed, code:%x", code); + QW_TASK_ELOG("qCreateExecTask failed, code:%x", code); QW_ERR_JRET(code); - } else { - QW_ERR_JRET(qwAddTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING, QW_EXIST_RET_ERR, NULL, NULL)); } - + QW_ERR_JRET(qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_SUCCESS)); queryRsped = true; DataSinkHandle sinkHandle = NULL; code = qExecTask(pTaskInfo, &sinkHandle); - if (code) { - qError("qExecTask failed, code:%x", code); + QW_TASK_ELOG("qExecTask failed, code:%x", code); QW_ERR_JRET(code); - } else { - QW_ERR_JRET(qwAddTaskHandlesToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, sinkHandle)); - QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED)); - } + } + + QW_ERR_JRET(qwAddTaskHandlesToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, sinkHandle)); _return: - if (queryRsped) { - code = qwCheckAndSendReadyRsp(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg, code); - } else { - code = qwBuildAndSendQueryRsp(pMsg, code); + if (code) { + rspCode = code; + } + + if (!queryRsped) { + code = qwBuildAndSendQueryRsp(pMsg, rspCode); + if (TSDB_CODE_SUCCESS == rspCode && code) { + rspCode = code; + } } int8_t status = 0; - if (TSDB_CODE_SUCCESS != code) { + if (TSDB_CODE_SUCCESS != rspCode) { status = JOB_TASK_STATUS_FAILED; } else { status = JOB_TASK_STATUS_PARTIAL_SUCCEED; } - qwQueryPostProcess(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, status, code); + qwQueryPostProcess(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, status, rspCode); + + if (queryRsped) { + qwCheckAndSendReadyRsp(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg); + } - QW_RET(code); + QW_RET(rspCode); } int32_t qWorkerProcessQueryContinueMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; int8_t status = 0; bool queryDone = false; - uint64_t sId, qId, tId; + SQueryContinueReq *req = (SQueryContinueReq *)pMsg->pCont; + bool needStop = false; + SQWTaskCtx *handles = NULL; - //TODO call executer to continue execute subquery - code = 0; - void *data = NULL; - queryDone = false; - //TODO call executer to continue execute subquery + QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, qWorkerMgmt, req->queryId, req->taskId, &handles)); + QW_LOCK(QW_WRITE, &handles->lock); + + qTaskInfo_t taskHandle = handles->taskHandle; + DataSinkHandle sinkHandle = handles->sinkHandle; + + QW_UNLOCK(QW_WRITE, &handles->lock); + qwReleaseTaskResCache(QW_READ, qWorkerMgmt); + QW_ERR_JRET(qwCheckAndProcessTaskDrop(qWorkerMgmt, req->sId, req->queryId, req->taskId, &needStop)); + if (needStop) { + qWarn("task need stop"); + + QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, qWorkerMgmt, req->queryId, req->taskId, &handles)); + QW_LOCK(QW_WRITE, &handles->lock); + if (handles->needRsp) { + qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_QRY_TASK_CANCELLED); + handles->needRsp = false; + } + QW_UNLOCK(QW_WRITE, &handles->lock); + qwReleaseTaskResCache(QW_READ, qWorkerMgmt); + + QW_ERR_RET(TSDB_CODE_QRY_TASK_CANCELLED); + } + + DataSinkHandle newHandle = NULL; + code = qExecTask(taskHandle, &newHandle); + if (code) { + qError("qExecTask failed, code:%x", code); + QW_ERR_JRET(code); + } + + if (sinkHandle != newHandle) { + qError("data sink mis-match"); + QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + +_return: + + QW_ERR_JRET(qwAcquireTaskCtx(QW_READ, qWorkerMgmt, req->queryId, req->taskId, &handles)); + QW_LOCK(QW_WRITE, &handles->lock); + + if (handles->needRsp) { + code = qwBuildAndSendQueryRsp(pMsg, code); + handles->needRsp = false; + } + handles->queryScheduled = false; + + QW_UNLOCK(QW_WRITE, &handles->lock); + qwReleaseTaskResCache(QW_READ, qWorkerMgmt); + if (TSDB_CODE_SUCCESS != code) { status = JOB_TASK_STATUS_FAILED; - } else if (queryDone) { - status = JOB_TASK_STATUS_SUCCEED; } else { status = JOB_TASK_STATUS_PARTIAL_SUCCEED; } - code = qwQueryPostProcess(qWorkerMgmt, sId, qId, tId, status, code); - + code = qwQueryPostProcess(qWorkerMgmt, req->sId, req->queryId, req->taskId, status, code); + QW_RET(code); } -int32_t qWorkerProcessSinkDataMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ +int32_t qWorkerProcessDataSinkMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ if (NULL == node || NULL == qWorkerMgmt || NULL == pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } @@ -1176,8 +1399,9 @@ int32_t qWorkerProcessSinkDataMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg){ if (NULL == msg || pMsg->contLen < sizeof(*msg)) { qError("invalid sink data msg"); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } + } + //dsScheduleProcess(); //TODO return TSDB_CODE_SUCCESS; diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index eaa79fd39ad7f8c69f5341bdf616e692f6d2a669..4962eab460faf5594c78efd40138aa1330fdd011 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -42,6 +42,11 @@ int32_t qwtStringToPlan(const char* str, SSubplan** subplan) { return 0; } +int32_t qwtPutReqToQueue(void *node, struct SRpcMsg *pMsg) { + return 0; +} + + void qwtRpcSendResponse(const SRpcMsg *pRsp) { if (TDMT_VND_TASKS_STATUS_RSP == pRsp->msgType) { SSchedulerStatusRsp *rsp = (SSchedulerStatusRsp *)pRsp->pCont; @@ -258,7 +263,7 @@ TEST(seqTest, normalCase) { stubSetStringToPlan(); stubSetRpcSendResponse(); - code = qWorkerInit(NULL, &mgmt); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); statusMsg.sId = htobe64(1); @@ -328,7 +333,7 @@ TEST(seqTest, cancelFirst) { stubSetStringToPlan(); stubSetRpcSendResponse(); - code = qWorkerInit(NULL, &mgmt); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); statusMsg.sId = htobe64(1); @@ -402,7 +407,7 @@ TEST(seqTest, randCase) { srand(time(NULL)); - code = qWorkerInit(NULL, &mgmt); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); int32_t t = 0; @@ -446,7 +451,7 @@ TEST(seqTest, multithreadRand) { srand(time(NULL)); - code = qWorkerInit(NULL, &mgmt); + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue); ASSERT_EQ(code, 0); pthread_attr_t thattr; diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index c83eba4232952fa72984251b96f4e06fde57e523..661beee5d5ebfe03d13eb5647fa3418d03d3cccf 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -36,11 +36,31 @@ enum { SCH_WRITE, }; +typedef struct SSchApiStat { + +} SSchApiStat; + +typedef struct SSchRuntimeStat { + +} SSchRuntimeStat; + +typedef struct SSchJobStat { + +} SSchJobStat; + +typedef struct SSchedulerStat { + SSchApiStat api; + SSchRuntimeStat runtime; + SSchJobStat job; +} SSchedulerStat; + + typedef struct SSchedulerMgmt { - uint64_t taskId; // sequential taksId - uint64_t sId; // schedulerId - SSchedulerCfg cfg; - SHashObj *jobs; // key: queryId, value: SQueryJob* + uint64_t taskId; // sequential taksId + uint64_t sId; // schedulerId + SSchedulerCfg cfg; + SHashObj *jobs; // key: queryId, value: SQueryJob* + SSchedulerStat stat; } SSchedulerMgmt; typedef struct SSchCallbackParam { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 94574762175cdf0c27718adfbf982e1bfdc163da..62ea28b811d15e9901fd0bb5d333fdffb1eb072c 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1462,34 +1462,37 @@ void scheduleFreeJob(void *job) { } SSchJob *pJob = job; + uint64_t queryId = pJob->queryId; - if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) { - SCH_JOB_ELOG("taosHashRemove job from list failed, may already freed, pJob:%p", pJob); - return; - } + if (SCH_GET_JOB_STATUS(pJob) > 0) { + if (0 != taosHashRemove(schMgmt.jobs, &pJob->queryId, sizeof(pJob->queryId))) { + SCH_JOB_ELOG("taosHashRemove job from list failed, may already freed, pJob:%p", pJob); + return; + } - schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); + schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); - SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref)); + SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref)); - while (true) { - int32_t ref = atomic_load_32(&pJob->ref); - if (0 == ref) { - break; - } else if (ref > 0) { - usleep(1); - } else { - assert(0); + while (true) { + int32_t ref = atomic_load_32(&pJob->ref); + if (0 == ref) { + break; + } else if (ref > 0) { + usleep(1); + } else { + assert(0); + } } - } - SCH_JOB_DLOG("job no ref now, status:%d", SCH_GET_JOB_STATUS(pJob)); + SCH_JOB_DLOG("job no ref now, status:%d", SCH_GET_JOB_STATUS(pJob)); - if (pJob->status == JOB_TASK_STATUS_EXECUTING) { - schCancelJob(pJob); - } + if (pJob->status == JOB_TASK_STATUS_EXECUTING) { + schCancelJob(pJob); + } - schDropJobAllTasks(pJob); + schDropJobAllTasks(pJob); + } pJob->subPlans = NULL; // it is a reference to pDag->pSubplans @@ -1515,6 +1518,8 @@ void scheduleFreeJob(void *job) { tfree(pJob->res); tfree(pJob); + + qDebug("QID:%"PRIx64" job freed", queryId); } void schedulerDestroy(void) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index ccf22173cbae9993e75240e4a59ef06c421321e2..d72c4920d51ee94cf1a36f188532c517dfcac0cc 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -79,6 +79,7 @@ void schtBuildQueryDag(SQueryDag *dag) { scanPlan->level = 1; scanPlan->pParents = taosArrayInit(1, POINTER_BYTES); scanPlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + scanPlan->msgType = TDMT_VND_QUERY; mergePlan->id.queryId = qId; mergePlan->id.templateId = 0x4444444444; @@ -89,6 +90,7 @@ void schtBuildQueryDag(SQueryDag *dag) { mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES); mergePlan->pParents = NULL; mergePlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + mergePlan->msgType = TDMT_VND_QUERY; SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); @@ -163,6 +165,11 @@ void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { } +void schtRpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) { + +} + + void schtSetPlanToString() { static Stub stub; @@ -190,6 +197,20 @@ void schtSetExecNode() { } } +void schtSetRpcSendRequest() { + static Stub stub; + stub.set(rpcSendRequest, schtRpcSendRequest); + { + AddrAny any("libtransport.so"); + std::map result; + any.get_global_func_addr_dynsym("^rpcSendRequest$", result); + for (const auto& f : result) { + stub.set(f.second, schtRpcSendRequest); + } + } +} + + void *schtSendRsp(void *param) { SSchJob *job = NULL; int32_t code = 0; diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index c4eeef5df20f23e8d2c557bc7e91a8ce2442032c..61d781210c7eb0239ce447a757a9a25c54a6775d 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -27,4 +27,11 @@ if (${BUILD_WITH_UV}) add_definitions(-DUSE_UV) endif(${BUILD_WITH_UV}) +if (${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) + + + + diff --git a/source/libs/transport/inc/rpcHead.h b/source/libs/transport/inc/rpcHead.h index 7317d84af1671c06452b6b3f4525675414e160a5..5ddf1a83c96271abba9aefc254b9cee442c375cc 100644 --- a/source/libs/transport/inc/rpcHead.h +++ b/source/libs/transport/inc/rpcHead.h @@ -21,10 +21,6 @@ extern "C" { #endif -#ifdef USE_UV - -#else - #define RPC_CONN_TCP 2 extern int tsRpcOverhead; @@ -75,7 +71,6 @@ typedef struct { } SRpcDigest; #pragma pack(pop) -#endif #ifdef __cplusplus } diff --git a/source/libs/transport/inc/rpcTcp.h b/source/libs/transport/inc/rpcTcp.h index 5e5c43a1db95a2886552db9c0e65df83115b2cd3..ad42307516462521055575d11bef38273b125fe1 100644 --- a/source/libs/transport/inc/rpcTcp.h +++ b/source/libs/transport/inc/rpcTcp.h @@ -21,8 +21,6 @@ extern "C" { #endif -#ifdef USE_UV -#else void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); void taosStopTcpServer(void *param); void taosCleanUpTcpServer(void *param); @@ -35,8 +33,6 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin void taosCloseTcpConnection(void *chandle); int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); -#endif - #ifdef __cplusplus } #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 9809f7ee1af24f3cbe426c7e67f62f7f0ebebb42..f93753cfe9a20d39cff4a313dc1b81866ebecd73 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -16,15 +16,65 @@ #ifndef _TD_TRANSPORT_INT_H_ #define _TD_TRANSPORT_INT_H_ +#include "rpcHead.h" #ifdef __cplusplus extern "C" { #endif #ifdef USE_UV -#else +#include +typedef void *queue[2]; + +/* Private macros. */ +#define QUEUE_NEXT(q) (*(queue **)&((*(q))[0])) +#define QUEUE_PREV(q) (*(queue **)&((*(q))[1])) + +#define QUEUE_PREV_NEXT(q) (QUEUE_NEXT(QUEUE_PREV(q))) +#define QUEUE_NEXT_PREV(q) (QUEUE_PREV(QUEUE_NEXT(q))) + +/* Initialize an empty queue. */ +#define QUEUE_INIT(q) \ + { \ + QUEUE_NEXT(q) = (q); \ + QUEUE_PREV(q) = (q); \ + } + +/* Return true if the queue has no element. */ +#define QUEUE_IS_EMPTY(q) ((const queue *)(q) == (const queue *)QUEUE_NEXT(q)) + +/* Insert an element at the back of a queue. */ +#define QUEUE_PUSH(q, e) \ + { \ + QUEUE_NEXT(e) = (q); \ + QUEUE_PREV(e) = QUEUE_PREV(q); \ + QUEUE_PREV_NEXT(e) = (e); \ + QUEUE_PREV(q) = (e); \ + } + +/* Remove the given element from the queue. Any element can be removed at any * + * time. */ +#define QUEUE_REMOVE(e) \ + { \ + QUEUE_PREV_NEXT(e) = QUEUE_NEXT(e); \ + QUEUE_NEXT_PREV(e) = QUEUE_PREV(e); \ + } + +/* Return the element at the front of the queue. */ +#define QUEUE_HEAD(q) (QUEUE_NEXT(q)) + +/* Return the element at the back of the queue. */ +#define QUEUE_TAIL(q) (QUEUE_PREV(q)) + +/* Iterate over the element of a queue. * Mutating the queue while iterating + * results in undefined behavior. */ +#define QUEUE_FOREACH(q, e) for ((q) = QUEUE_NEXT(e); (q) != (e); (q) = QUEUE_NEXT(q)) + +/* Return the structure holding the given element. */ +#define QUEUE_DATA(e, type, field) ((type *)((void *)((char *)(e)-offsetof(type, field)))) + +#endif // USE_LIBUV -#endif #ifdef __cplusplus } #endif diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c index 40767d2ba53c27cc183c29305837d041673320d2..1db2808126a162e4b15305ff80057955de1025a3 100644 --- a/source/libs/transport/src/rpcCache.c +++ b/source/libs/transport/src/rpcCache.c @@ -22,9 +22,6 @@ #include "ttimer.h" #include "tutil.h" -#ifdef USE_UV - -#else typedef struct SConnHash { char fqdn[TSDB_FQDN_LEN]; uint16_t port; @@ -295,4 +292,3 @@ static void rpcUnlockCache(int64_t *lockedBy) { assert(false); } } -#endif diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 3095ddb9d254e67692a26874040c7ad3580091d7..f381768a34de1a6c46a55f5aeaa7d495f9d4d655 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -13,9 +13,6 @@ * along with this program. If not, see . */ -#ifdef USE_UV -#include -#endif #include "lz4.h" #include "os.h" #include "rpcCache.h" @@ -30,11 +27,23 @@ #include "tmd5.h" #include "tmempool.h" #include "tmsg.h" +#include "transportInt.h" #include "tref.h" #include "trpc.h" #include "ttimer.h" #include "tutil.h" +static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; + +int tsRpcMaxUdpSize = 15000; // bytes +int tsProgressTimer = 100; +// not configurable +int tsRpcMaxRetry; +int tsRpcHeadSize; +int tsRpcOverhead; + +#ifndef USE_UV + typedef struct { int sessions; // number of sessions allowed int numOfThreads; // number of threads to process incoming messages @@ -50,235 +59,21 @@ typedef struct { char secret[TSDB_PASSWORD_LEN]; // secret for the link char ckey[TSDB_PASSWORD_LEN]; // ciphering key - void (*cfp)(void* parent, SRpcMsg*, SEpSet*); - int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + void (*cfp)(void *parent, SRpcMsg *, SEpSet *); + int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); int32_t refCount; - void* parent; - void* idPool; // handle to ID pool - void* tmrCtrl; // handle to timer - SHashObj* hash; // handle returned by hash utility - void* tcphandle; // returned handle from TCP initialization - void* udphandle; // returned handle from UDP initialization - void* pCache; // connection cache + void * parent; + void * idPool; // handle to ID pool + void * tmrCtrl; // handle to timer + SHashObj * hash; // handle returned by hash utility + void * tcphandle; // returned handle from TCP initialization + void * udphandle; // returned handle from UDP initialization + void * pCache; // connection cache pthread_mutex_t mutex; - struct SRpcConn* connList; // connection list + struct SRpcConn *connList; // connection list } SRpcInfo; -#ifdef USE_UV - -#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) - -typedef struct SThreadObj { - pthread_t thread; - uv_pipe_t* pipe; - uv_loop_t* loop; - uv_async_t* workerAsync; // - int fd; -} SThreadObj; - -typedef struct SServerObj { - uv_tcp_t server; - uv_loop_t* loop; - int workerIdx; - int numOfThread; - SThreadObj** pThreadObj; - uv_pipe_t** pipe; -} SServerObj; - -typedef struct SConnCtx { - uv_tcp_t* pClient; - uv_timer_t* pTimer; - uv_async_t* pWorkerAsync; - int ref; -} SConnCtx; - -static void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); -static void onTimeout(uv_timer_t* handle); -static void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); -static void onWrite(uv_write_t* req, int status); -static void onAccept(uv_stream_t* stream, int status); -void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); -static void workerAsyncCB(uv_async_t* handle); -static void* workerThread(void* arg); - -int32_t rpcInit() { return -1; } -void rpcCleanup() { return; }; -void* rpcOpen(const SRpcInit* pInit) { - SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); - if (pRpc == NULL) { - return NULL; - } - if (pInit->label) { - tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); - } - pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; - - SServerObj* srv = calloc(1, sizeof(SServerObj)); - srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - srv->numOfThread = pRpc->numOfThreads; - srv->workerIdx = 0; - srv->pThreadObj = (SThreadObj**)calloc(srv->numOfThread, sizeof(SThreadObj*)); - srv->pipe = (uv_pipe_t**)calloc(srv->numOfThread, sizeof(uv_pipe_t*)); - uv_loop_init(srv->loop); - - for (int i = 0; i < srv->numOfThread; i++) { - srv->pThreadObj[i] = (SThreadObj*)calloc(1, sizeof(SThreadObj)); - srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); - int fds[2]; - if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { - return NULL; - } - uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); - uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write - - srv->pThreadObj[i]->fd = fds[0]; - srv->pThreadObj[i]->pipe = &(srv->pipe[i][1]); // init read - int err = pthread_create(&(srv->pThreadObj[i]->thread), NULL, workerThread, (void*)(srv->pThreadObj[i])); - if (err == 0) { - tError("sucess to create worker thread %d", i); - // printf("thread %d create\n", i); - } else { - tError("failed to create worker thread %d", i); - return NULL; - } - } - uv_tcp_init(srv->loop, &srv->server); - struct sockaddr_in bind_addr; - uv_ip4_addr("0.0.0.0", pInit->localPort, &bind_addr); - uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); - int err = 0; - if ((err = uv_listen((uv_stream_t*)&srv->server, 128, onAccept)) != 0) { - tError("Listen error %s\n", uv_err_name(err)); - return NULL; - } - uv_run(srv->loop, UV_RUN_DEFAULT); - - return pRpc; -} -void rpcClose(void* arg) { return; } -void* rpcMallocCont(int contLen) { return NULL; } -void rpcFreeCont(void* cont) { return; } -void* rpcReallocCont(void* ptr, int contLen) { return NULL; } - -void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } - -void rpcSendResponse(const SRpcMsg* pMsg) {} - -void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} -int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } -void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } -int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } -void rpcCancelRequest(int64_t rid) { return; } - -void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - buf->base = malloc(suggested_size); - buf->len = suggested_size; -} - -void onTimeout(uv_timer_t* handle) { - // opt - tDebug("time out"); -} -void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { - // opt - tDebug("data already was read on a stream"); -} - -void onWrite(uv_write_t* req, int status) { - // opt - if (req) tDebug("data already was written on stream"); -} - -void workerAsyncCB(uv_async_t* handle) { - // opt - SThreadObj* pObj = container_of(handle, SThreadObj, workerAsync); -} -void onAccept(uv_stream_t* stream, int status) { - if (status == -1) { - return; - } - SServerObj* pObj = container_of(stream, SServerObj, server); - tDebug("new conntion accepted by main server, dispatch to one worker thread"); - - uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); - uv_tcp_init(pObj->loop, cli); - if (uv_accept(stream, (uv_stream_t*)cli) == 0) { - uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); - - uv_buf_t buf = uv_buf_init("a", 1); - // despatch to worker thread - pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; - uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, onWrite); - } else { - uv_close((uv_handle_t*)cli, NULL); - } -} -void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { - if (nread < 0) { - if (nread != UV_EOF) { - tError("read error %s", uv_err_name(nread)); - } - // TODO(log other failure reason) - uv_close((uv_handle_t*)q, NULL); - return; - } - SThreadObj* pObj = (SThreadObj*)container_of(q, struct SThreadObj, pipe); - - uv_pipe_t* pipe = (uv_pipe_t*)q; - if (!uv_pipe_pending_count(pipe)) { - tError("No pending count"); - return; - } - uv_handle_type pending = uv_pipe_pending_type(pipe); - assert(pending == UV_TCP); - - SConnCtx* pConn = malloc(sizeof(SConnCtx)); - /* init conn timer*/ - pConn->pTimer = malloc(sizeof(uv_timer_t)); - uv_timer_init(pObj->loop, pConn->pTimer); - - pConn->pClient = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); - pConn->pWorkerAsync = pObj->workerAsync; // thread safty - uv_tcp_init(pObj->loop, pConn->pClient); - - if (uv_accept(q, (uv_stream_t*)(pConn->pClient)) == 0) { - uv_os_fd_t fd; - uv_fileno((const uv_handle_t*)pConn->pClient, &fd); - tDebug("new connection created: %d", fd); - uv_timer_start(pConn->pTimer, onTimeout, 10, 0); - uv_read_start((uv_stream_t*)(pConn->pClient), allocBuffer, onRead); - } else { - uv_timer_stop(pConn->pTimer); - free(pConn->pTimer); - uv_close((uv_handle_t*)pConn->pClient, NULL); - free(pConn->pClient); - free(pConn); - } -} - -void* workerThread(void* arg) { - SThreadObj* pObj = (SThreadObj*)arg; - int fd = pObj->fd; - pObj->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); - uv_loop_init(pObj->loop); - - uv_pipe_init(pObj->loop, pObj->pipe, 1); - uv_pipe_open(pObj->pipe, fd); - - pObj->workerAsync = malloc(sizeof(uv_async_t)); - uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCB); - uv_read_start((uv_stream_t*)pObj->pipe, allocBuffer, onConnection); -} -#else - -#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) -#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) -#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) -#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) -#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) -#define rpcIsReq(type) (type & 1U) - typedef struct { SRpcInfo * pRpc; // associated SRpcInfo SEpSet epSet; // ip list provided by app @@ -299,6 +94,13 @@ typedef struct { char msg[0]; // RpcHead starts from here } SRpcReqContext; +#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) +#define rpcHeadFromCont(cont) ((SRpcHead *)((char *)cont - sizeof(SRpcHead))) +#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) +#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) +#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) +#define rpcIsReq(type) (type & 1U) + typedef struct SRpcConn { char info[48]; // debug info: label + pConn + ahandle int sid; // session ID @@ -336,15 +138,6 @@ typedef struct SRpcConn { SRpcReqContext *pContext; // request context } SRpcConn; -static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; - -int tsRpcMaxUdpSize = 15000; // bytes -int tsProgressTimer = 100; -// not configurable -int tsRpcMaxRetry; -int tsRpcHeadSize; -int tsRpcOverhead; - static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; // static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; @@ -442,7 +235,8 @@ void *rpcOpen(const SRpcInit *pInit) { pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; - if (pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); + if (pInit->label) tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); + pRpc->connType = pInit->connType; if (pRpc->connType == TAOS_CONN_CLIENT) { pRpc->numOfThreads = pInit->numOfThreads; diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index 59d223153a20412a90bc21a043df764922b8fcb3..56dd8cce2512eca08e62a3f6d393253e86612c29 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -14,9 +14,6 @@ */ #include "rpcTcp.h" -#ifdef USE_UV -#include -#endif #include "os.h" #include "rpcHead.h" #include "rpcLog.h" @@ -24,9 +21,6 @@ #include "taoserror.h" #include "tutil.h" -#ifdef USE_UV - -#else typedef struct SFdObj { void * signature; SOCKET fd; // TCP socket FD @@ -665,5 +659,3 @@ static void taosFreeFdObj(SFdObj *pFdObj) { tfree(pFdObj); } - -#endif diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index 79956cc98db038c6e6e06a06fdbd7cb947c4ecef..b57cf57c55b5a0ae4ddb60f2883e8524bebb960d 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -22,9 +22,6 @@ #include "ttimer.h" #include "tutil.h" -#ifdef USE_UV -// no support upd currently -#else #define RPC_MAX_UDP_CONNS 256 #define RPC_MAX_UDP_PKTS 1000 #define RPC_UDP_BUF_TIME 5 // mseconds @@ -260,4 +257,3 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c return ret; } -#endif diff --git a/source/libs/transport/src/transport.c b/source/libs/transport/src/transport.c index f2f48bbc8a69a022d0fc6b8a88c5a9a55d0b4ad6..93bbaf282002842a7344dbf650ec856ace42eff0 100644 --- a/source/libs/transport/src/transport.c +++ b/source/libs/transport/src/transport.c @@ -12,3 +12,735 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +#ifdef USE_UV + +#include +#include "lz4.h" +#include "os.h" +#include "rpcCache.h" +#include "rpcHead.h" +#include "rpcLog.h" +#include "rpcTcp.h" +#include "rpcUdp.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" +#include "tidpool.h" +#include "tmd5.h" +#include "tmempool.h" +#include "tmsg.h" +#include "transportInt.h" +#include "tref.h" +#include "trpc.h" +#include "ttimer.h" +#include "tutil.h" + +#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) +#define RPC_RESERVE_SIZE (sizeof(SRpcReqContext)) +static const char* notify = "a"; + +typedef struct { + int sessions; // number of sessions allowed + int numOfThreads; // number of threads to process incoming messages + int idleTime; // milliseconds; + uint16_t localPort; + int8_t connType; + int index; // for UDP server only, round robin for multiple threads + char label[TSDB_LABEL_LEN]; + + char user[TSDB_UNI_LEN]; // meter ID + char spi; // security parameter index + char encrypt; // encrypt algorithm + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key + + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); + int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + + int32_t refCount; + void* parent; + void* idPool; // handle to ID pool + void* tmrCtrl; // handle to timer + SHashObj* hash; // handle returned by hash utility + void* tcphandle; // returned handle from TCP initialization + void* udphandle; // returned handle from UDP initialization + void* pCache; // connection cache + pthread_mutex_t mutex; + struct SRpcConn* connList; // connection list +} SRpcInfo; + +typedef struct { + SRpcInfo* pRpc; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app + struct SRpcConn* pConn; // pConn allocated + tmsg_t msgType; // message type + uint8_t* pCont; // content provided by app + int32_t contLen; // content length + int32_t code; // error code + int16_t numOfTry; // number of try for different servers + int8_t oldInUse; // server EP inUse passed by app + int8_t redirect; // flag to indicate redirect + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + SRpcMsg* pRsp; // for synchronous API + tsem_t* pSem; // for synchronous API + SEpSet* pSet; // for synchronous API + char msg[0]; // RpcHead starts from here +} SRpcReqContext; + +typedef struct SThreadObj { + pthread_t thread; + uv_pipe_t* pipe; + int fd; + uv_loop_t* loop; + uv_async_t* workerAsync; // + queue conn; + pthread_mutex_t connMtx; + void* shandle; +} SThreadObj; + +#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) +#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) +#define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) +#define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) +#define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) +#define rpcIsReq(type) (type & 1U) + +typedef struct SServerObj { + pthread_t thread; + uv_tcp_t server; + uv_loop_t* loop; + int workerIdx; + int numOfThread; + SThreadObj** pThreadObj; + uv_pipe_t** pipe; + uint32_t ip; + uint32_t port; +} SServerObj; + +typedef struct SConnBuffer { + char* buf; + int len; + int cap; + int left; +} SConnBuffer; + +typedef struct SRpcConn { + uv_tcp_t* pTcp; + uv_write_t* pWriter; + uv_timer_t* pTimer; + + uv_async_t* pWorkerAsync; + queue queue; + int ref; + int persist; // persist connection or not + SConnBuffer connBuf; // read buf, + SConnBuffer writeBuf; // write buf + int count; + void* shandle; // rpc init + void* ahandle; // + void* hostThread; + // del later + char secured; + int spi; + char info[64]; + char user[TSDB_UNI_LEN]; // user ID for the link + char secret[TSDB_PASSWORD_LEN]; + char ckey[TSDB_PASSWORD_LEN]; // ciphering key +} SRpcConn; + +// auth function +static int uvAuthMsg(SRpcConn* pConn, char* msg, int msgLen); +static int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey); +static void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey); +static int rpcAddAuthPart(SRpcConn* pConn, char* msg, int msgLen); +// compress data +static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); +static SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead); + +static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +static void uvOnTimeoutCb(uv_timer_t* handle); +static void uvOnWriteCb(uv_write_t* req, int status); +static void uvOnAcceptCb(uv_stream_t* stream, int status); +static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); +static void uvWorkerAsyncCb(uv_async_t* handle); + +static SRpcConn* connCreate(); +static void connDestroy(SRpcConn* conn); +static void uvConnDestroy(uv_handle_t* handle); + +static void* workerThread(void* arg); +static void* acceptThread(void* arg); + +void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); +void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); + +void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { + SServerObj* srv = calloc(1, sizeof(SServerObj)); + srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + srv->numOfThread = numOfThreads; + srv->workerIdx = 0; + srv->pThreadObj = (SThreadObj**)calloc(srv->numOfThread, sizeof(SThreadObj*)); + srv->pipe = (uv_pipe_t**)calloc(srv->numOfThread, sizeof(uv_pipe_t*)); + srv->ip = ip; + srv->port = port; + uv_loop_init(srv->loop); + + for (int i = 0; i < srv->numOfThread; i++) { + SThreadObj* thrd = (SThreadObj*)calloc(1, sizeof(SThreadObj)); + srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); + int fds[2]; + if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { + return NULL; + } + uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); + uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write + + thrd->shandle = shandle; + thrd->fd = fds[0]; + thrd->pipe = &(srv->pipe[i][1]); // init read + int err = pthread_create(&(thrd->thread), NULL, workerThread, (void*)(thrd)); + if (err == 0) { + tDebug("sucess to create worker-thread %d", i); + // printf("thread %d create\n", i); + } else { + // TODO: clear all other resource later + tError("failed to create worker-thread %d", i); + } + srv->pThreadObj[i] = thrd; + } + + int err = pthread_create(&srv->thread, NULL, acceptThread, (void*)srv); + if (err == 0) { + tDebug("success to create accept-thread"); + } else { + // clear all resource later + } + + return srv; +} +void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + /* + * formate of data buffer: + * |<-------SRpcReqContext------->|<------------data read from socket----------->| + */ + static const int CAPACITY = 1024; + + SRpcConn* ctx = handle->data; + SConnBuffer* pBuf = &ctx->connBuf; + if (pBuf->cap == 0) { + pBuf->buf = (char*)calloc(CAPACITY + RPC_RESERVE_SIZE, sizeof(char)); + pBuf->len = 0; + pBuf->cap = CAPACITY; + pBuf->left = -1; + + buf->base = pBuf->buf + RPC_RESERVE_SIZE; + buf->len = CAPACITY; + } else { + if (pBuf->len >= pBuf->cap) { + if (pBuf->left == -1) { + pBuf->cap *= 2; + pBuf->buf = realloc(pBuf->buf, pBuf->cap + RPC_RESERVE_SIZE); + } else if (pBuf->len + pBuf->left > pBuf->cap) { + pBuf->cap = pBuf->len + pBuf->left; + pBuf->buf = realloc(pBuf->buf, pBuf->len + pBuf->left + RPC_RESERVE_SIZE); + } + } + buf->base = pBuf->buf + pBuf->len + RPC_RESERVE_SIZE; + buf->len = pBuf->cap - pBuf->len; + } +} +// check data read from socket completely or not +// +static bool isReadAll(SConnBuffer* data) { + // TODO(yihao): handle pipeline later + SRpcHead rpcHead; + int32_t headLen = sizeof(rpcHead); + if (data->len >= headLen) { + memcpy((char*)&rpcHead, data->buf + RPC_RESERVE_SIZE, headLen); + int32_t msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); + if (msgLen > data->len) { + data->left = msgLen - data->len; + return false; + } else { + return true; + } + } else { + return false; + } +} +static void uvDoProcess(SRecvInfo* pRecv) { + SRpcHead* pHead = (SRpcHead*)pRecv->msg; + SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle; + SRpcConn* pConn = pRecv->thandle; + + tDump(pRecv->msg, pRecv->msgLen); + + terrno = 0; + SRpcReqContext* pContest; + + // do auth and check +} +static int uvAuthMsg(SRpcConn* pConn, char* msg, int len) { + SRpcHead* pHead = (SRpcHead*)msg; + int code = 0; + + if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { + // secured link, or no authentication + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); + // tTrace("%s, secured link, no auth is required", pConn->info); + return 0; + } + + if (!rpcIsReq(pHead->msgType)) { + // for response, if code is auth failure, it shall bypass the auth process + code = htonl(pHead->code); + if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || + code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || + code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); + // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); + return 0; + } + } + + code = 0; + if (pHead->spi == pConn->spi) { + // authentication + SRpcDigest* pDigest = (SRpcDigest*)((char*)pHead + len - sizeof(SRpcDigest)); + + int32_t delta; + delta = (int32_t)htonl(pDigest->timeStamp); + delta -= (int32_t)taosGetTimestampSec(); + if (abs(delta) > 900) { + tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); + code = TSDB_CODE_RPC_INVALID_TIME_STAMP; + } else { + if (rpcAuthenticateMsg(pHead, len - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { + // tDebug("%s, authentication failed, msg discarded", pConn->info); + code = TSDB_CODE_RPC_AUTH_FAILURE; + } else { + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); + if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client + // tTrace("%s, message is authenticated", pConn->info); + } + } + } else { + tDebug("%s, auth spi:%d not matched with received:%d", pConn->info, pConn->spi, pHead->spi); + code = pHead->spi ? TSDB_CODE_RPC_AUTH_FAILURE : TSDB_CODE_RPC_AUTH_REQUIRED; + } + + return code; +} +// refers specifically to query or insert timeout +static void uvHandleActivityTimeout(uv_timer_t* handle) { + // impl later + SRpcConn* conn = handle->data; +} +static void uvProcessData(SRpcConn* pConn) { + SRecvInfo info; + SRecvInfo* p = &info; + SConnBuffer* pBuf = &pConn->connBuf; + p->msg = pBuf->buf + RPC_RESERVE_SIZE; + p->msgLen = pBuf->len; + p->ip = 0; + p->port = 0; + p->shandle = pConn->shandle; // + p->thandle = pConn; + p->chandle = NULL; + + // + SRpcHead* pHead = (SRpcHead*)p->msg; + assert(rpcIsReq(pHead->msgType)); + + SRpcInfo* pRpc = (SRpcInfo*)p->shandle; + pConn->ahandle = (void*)pHead->ahandle; + // auth here + + int8_t code = uvAuthMsg(pConn, (char*)pHead, p->msgLen); + if (code != 0) { + terrno = code; + return; + } + pHead->code = htonl(pHead->code); + + SRpcMsg rpcMsg; + + pHead = rpcDecompressRpcMsg(pHead); + rpcMsg.contLen = rpcContLenFromMsg(pHead->msgLen); + rpcMsg.pCont = pHead->content; + rpcMsg.msgType = pHead->msgType; + rpcMsg.code = pHead->code; + rpcMsg.ahandle = pConn->ahandle; + rpcMsg.handle = pConn; + + (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); + uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime, 0); + // auth + // validate msg type +} +void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { + // opt + SRpcConn* ctx = cli->data; + SConnBuffer* pBuf = &ctx->connBuf; + if (nread > 0) { + pBuf->len += nread; + if (isReadAll(pBuf)) { + tDebug("alread read complete packet"); + uvProcessData(ctx); + } else { + tDebug("read half packet, continue to read"); + } + return; + } + if (terrno != 0) { + // handle err code + } + + if (nread != UV_EOF) { + tDebug("Read error %s\n", uv_err_name(nread)); + } + uv_close((uv_handle_t*)cli, uvConnDestroy); +} +void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + buf->base = malloc(sizeof(char)); + buf->len = 2; +} + +void uvOnTimeoutCb(uv_timer_t* handle) { + // opt + tDebug("time out"); +} + +void uvOnWriteCb(uv_write_t* req, int status) { + SRpcConn* conn = req->data; + if (status == 0) { + tDebug("data already was written on stream"); + } else { + connDestroy(conn); + } + // opt +} + +void uvWorkerAsyncCb(uv_async_t* handle) { + SThreadObj* pThrd = container_of(handle, SThreadObj, workerAsync); + SRpcConn* conn = NULL; + + // opt later + pthread_mutex_lock(&pThrd->connMtx); + if (!QUEUE_IS_EMPTY(&pThrd->conn)) { + queue* head = QUEUE_HEAD(&pThrd->conn); + conn = QUEUE_DATA(head, SRpcConn, queue); + QUEUE_REMOVE(&conn->queue); + } + pthread_mutex_unlock(&pThrd->connMtx); + if (conn == NULL) { + tError("except occurred, do nothing"); + return; + } + uv_buf_t wb = uv_buf_init(conn->writeBuf.buf, conn->writeBuf.len); + uv_write(conn->pWriter, (uv_stream_t*)conn->pTcp, &wb, 1, uvOnWriteCb); +} + +void uvOnAcceptCb(uv_stream_t* stream, int status) { + if (status == -1) { + return; + } + SServerObj* pObj = container_of(stream, SServerObj, server); + + uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pObj->loop, cli); + + if (uv_accept(stream, (uv_stream_t*)cli) == 0) { + uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); + + uv_buf_t buf = uv_buf_init((char*)notify, strlen(notify)); + + pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; + tDebug("new conntion accepted by main server, dispatch to %dth worker-thread", pObj->workerIdx); + uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, uvOnWriteCb); + } else { + uv_close((uv_handle_t*)cli, NULL); + } +} +void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { + tDebug("connection coming"); + if (nread < 0) { + if (nread != UV_EOF) { + tError("read error %s", uv_err_name(nread)); + } + // TODO(log other failure reason) + uv_close((uv_handle_t*)q, NULL); + return; + } + // free memory allocated by + assert(nread == strlen(notify)); + assert(buf->base[0] == notify[0]); + free(buf->base); + + SThreadObj* pThrd = q->data; + + uv_pipe_t* pipe = (uv_pipe_t*)q; + if (!uv_pipe_pending_count(pipe)) { + tError("No pending count"); + return; + } + + uv_handle_type pending = uv_pipe_pending_type(pipe); + assert(pending == UV_TCP); + + SRpcConn* pConn = connCreate(); + pConn->shandle = pThrd->shandle; + /* init conn timer*/ + pConn->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pThrd->loop, pConn->pTimer); + pConn->pTimer->data = pConn; + + pConn->hostThread = pThrd; + pConn->pWorkerAsync = pThrd->workerAsync; // thread safty + + // init client handle + pConn->pTcp = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pThrd->loop, pConn->pTcp); + pConn->pTcp->data = pConn; + + // init write request, just + pConn->pWriter = calloc(1, sizeof(uv_write_t)); + pConn->pWriter->data = pConn; + + if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { + uv_os_fd_t fd; + uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); + tDebug("new connection created: %d", fd); + uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); + } else { + connDestroy(pConn); + } +} + +void* acceptThread(void* arg) { + // opt + SServerObj* srv = (SServerObj*)arg; + uv_tcp_init(srv->loop, &srv->server); + + struct sockaddr_in bind_addr; + + uv_ip4_addr("0.0.0.0", srv->port, &bind_addr); + uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); + int err = 0; + if ((err = uv_listen((uv_stream_t*)&srv->server, 128, uvOnAcceptCb)) != 0) { + tError("Listen error %s\n", uv_err_name(err)); + return NULL; + } + uv_run(srv->loop, UV_RUN_DEFAULT); +} +void* workerThread(void* arg) { + SThreadObj* pThrd = (SThreadObj*)arg; + + pThrd->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + uv_loop_init(pThrd->loop); + + uv_pipe_init(pThrd->loop, pThrd->pipe, 1); + uv_pipe_open(pThrd->pipe, pThrd->fd); + + pThrd->pipe->data = pThrd; + + QUEUE_INIT(&pThrd->conn); + + pThrd->workerAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pThrd->loop, pThrd->workerAsync, uvWorkerAsyncCb); + + uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); + uv_run(pThrd->loop, UV_RUN_DEFAULT); +} +static SRpcConn* connCreate() { + SRpcConn* pConn = (SRpcConn*)calloc(1, sizeof(SRpcConn)); + return pConn; +} +static void connDestroy(SRpcConn* conn) { + if (conn == NULL) { + return; + } + uv_timer_stop(conn->pTimer); + free(conn->pTimer); + uv_close((uv_handle_t*)conn->pTcp, NULL); + free(conn->connBuf.buf); + free(conn->pTcp); + free(conn->pWriter); + free(conn); + // handle +} +static void uvConnDestroy(uv_handle_t* handle) { + SRpcConn* conn = handle->data; + connDestroy(conn); +} +void* rpcOpen(const SRpcInit* pInit) { + SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); + if (pRpc == NULL) { + return NULL; + } + if (pInit->label) { + tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); + } + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + pRpc->tcphandle = taosInitServer(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + return pRpc; +} +void rpcClose(void* arg) { return; } +void* rpcMallocCont(int contLen) { return NULL; } +void rpcFreeCont(void* cont) { return; } +void* rpcReallocCont(void* ptr, int contLen) { return NULL; } + +void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { + // impl later + return; +} + +void rpcSendResponse(const SRpcMsg* pMsg) { + SRpcConn* pConn = pMsg->handle; + SThreadObj* pThrd = pConn->hostThread; + + // opt later + pthread_mutex_lock(&pThrd->connMtx); + QUEUE_PUSH(&pThrd->conn, &pConn->queue); + pthread_mutex_unlock(&pThrd->connMtx); + + uv_async_send(pConn->pWorkerAsync); +} + +void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} +int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } +void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } +int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } +void rpcCancelRequest(int64_t rid) { return; } + +static int rpcAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { + T_MD5_CTX context; + int ret = -1; + + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t*)pMsg, msgLen); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); + + if (memcmp(context.digest, pAuth, sizeof(context.digest)) == 0) ret = 0; + + return ret; +} +static void rpcBuildAuthHead(void* pMsg, int msgLen, void* pAuth, void* pKey) { + T_MD5_CTX context; + + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Update(&context, (uint8_t*)pMsg, msgLen); + tMD5Update(&context, (uint8_t*)pKey, TSDB_PASSWORD_LEN); + tMD5Final(&context); + + memcpy(pAuth, context.digest, sizeof(context.digest)); +} + +static int rpcAddAuthPart(SRpcConn* pConn, char* msg, int msgLen) { + SRpcHead* pHead = (SRpcHead*)msg; + + if (pConn->spi && pConn->secured == 0) { + // add auth part + pHead->spi = pConn->spi; + SRpcDigest* pDigest = (SRpcDigest*)(msg + msgLen); + pDigest->timeStamp = htonl(taosGetTimestampSec()); + msgLen += sizeof(SRpcDigest); + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + rpcBuildAuthHead(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret); + } else { + pHead->spi = 0; + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + } + + return msgLen; +} + +static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { + SRpcHead* pHead = rpcHeadFromCont(pCont); + int32_t finalLen = 0; + int overhead = sizeof(SRpcComp); + + if (!NEEDTO_COMPRESSS_MSG(contLen)) { + return contLen; + } + + char* buf = malloc(contLen + overhead + 8); // 8 extra bytes + if (buf == NULL) { + tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); + return contLen; + } + + int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead); + tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead); + + /* + * only the compressed size is less than the value of contLen - overhead, the compression is applied + * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message + */ + if (compLen > 0 && compLen < contLen - overhead) { + SRpcComp* pComp = (SRpcComp*)pCont; + pComp->reserved = 0; + pComp->contLen = htonl(contLen); + memcpy(pCont + overhead, buf, compLen); + + pHead->comp = 1; + tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen); + finalLen = compLen + overhead; + } else { + finalLen = contLen; + } + + free(buf); + return finalLen; +} + +static SRpcHead* rpcDecompressRpcMsg(SRpcHead* pHead) { + int overhead = sizeof(SRpcComp); + SRpcHead* pNewHead = NULL; + uint8_t* pCont = pHead->content; + SRpcComp* pComp = (SRpcComp*)pHead->content; + + if (pHead->comp) { + // decompress the content + assert(pComp->reserved == 0); + int contLen = htonl(pComp->contLen); + + // prepare the temporary buffer to decompress message + char* temp = (char*)malloc(contLen + RPC_MSG_OVERHEAD); + pNewHead = (SRpcHead*)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext + + if (pNewHead) { + int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead; + int origLen = LZ4_decompress_safe((char*)(pCont + overhead), (char*)pNewHead->content, compLen, contLen); + assert(origLen == contLen); + + memcpy(pNewHead, pHead, sizeof(SRpcHead)); + pNewHead->msgLen = rpcMsgLenFromCont(origLen); + /// rpcFreeMsg(pHead); // free the compressed message buffer + pHead = pNewHead; + tTrace("decomp malloc mem:%p", temp); + } else { + tError("failed to allocate memory to decompress msg, contLen:%d", contLen); + } + } + + return pHead; +} +int32_t rpcInit(void) { + // impl later + return -1; +} + +void rpcCleanup(void) { + // impl later + return; +} +#endif diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..9e58bf08cdeb27ed6f6c6a187fc156bd586dd2a4 --- /dev/null +++ b/source/libs/transport/test/CMakeLists.txt @@ -0,0 +1,21 @@ +add_executable(transportTest "") +target_sources(transportTest + PRIVATE + "transportTests.cc" +) + +target_include_directories(transportTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +target_link_libraries (transportTest + os + util + common + gtest_main + transport +) + + diff --git a/source/libs/transport/test/transportTests.cc b/source/libs/transport/test/transportTests.cc new file mode 100644 index 0000000000000000000000000000000000000000..468aeba8a9adad92d730bc251b3416eddea7e60f --- /dev/null +++ b/source/libs/transport/test/transportTests.cc @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include + +#include "transportInt.h" +#include "trpc.h" + +using namespace std; + +int main() { + SRpcInit init = {.localPort = 6030, .label = "rpc", .numOfThreads = 5}; + void* p = rpcOpen(&init); + + while (1) { + std::cout << "cron task" << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(10 * 1000)); + } +} diff --git a/source/libs/transport/test/transportTests.cpp b/source/libs/transport/test/transportTests.cpp deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index cac80c0a5f578da85b1256cc0b6d4a5ec9687c7d..d63008008697b8d10c3ae0e4f0cb0e587bbb4e6d 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -36,7 +36,7 @@ void* tmemmem(char* haystack, int hlen, char* needle, int nlen) { char* limit; if (nlen == 0 || hlen < nlen) { - return false; + return NULL; } limit = haystack + hlen - nlen + 1; @@ -54,10 +54,12 @@ static inline int64_t walScanLogGetLastVer(SWal* pWal) { ASSERT(pWal->fileInfoSet != NULL); int sz = taosArrayGetSize(pWal->fileInfoSet); ASSERT(sz > 0); +#if 0 for (int i = 0; i < sz; i++) { SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); - } +#endif + SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz-1); char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); @@ -143,8 +145,6 @@ int walCheckAndRepairMeta(SWal* pWal) { SWalFileInfo fileInfo; memset(&fileInfo, -1, sizeof(SWalFileInfo)); sscanf(name, "%" PRId64 ".log", &fileInfo.firstVer); - //get lastVer - //get size taosArrayPush(pLogInfoArray, &fileInfo); } } @@ -158,60 +158,51 @@ int walCheckAndRepairMeta(SWal* pWal) { oldSz = taosArrayGetSize(pWal->fileInfoSet); } int newSz = taosArrayGetSize(pLogInfoArray); - // case 1. meta file not exist / cannot be parsed - if (oldSz < newSz) { + + if (oldSz > newSz) { + taosArrayPopFrontBatch(pWal->fileInfoSet, oldSz - newSz); + } else if (oldSz < newSz) { for (int i = oldSz; i < newSz; i++) { SWalFileInfo *pFileInfo = taosArrayGet(pLogInfoArray, i); taosArrayPush(pWal->fileInfoSet, pFileInfo); } - - pWal->writeCur = newSz - 1; - pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pLogInfoArray, 0))->firstVer; - pWal->vers.lastVer = walScanLogGetLastVer(pWal); - ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = pWal->vers.lastVer; - ASSERT(pWal->vers.lastVer != -1); - - int code = walSaveMeta(pWal); - if (code < 0) { - taosArrayDestroy(pLogInfoArray); - return -1; - } } - - // case 2. versions in meta not match log - // or some log not included in meta - // (e.g. program killed) - // - // case 3. other corrupt cases - // -#if 0 - int sz = taosArrayGetSize(pLogInfoArray); - for (int i = 0; i < sz; i++) { - SWalFileInfo* pFileInfo = taosArrayGet(pLogInfoArray, i); - if (i == 0 && pFileInfo->firstVer != walGetFirstVer(pWal)) { - //repair - } - - if (i > 0) { - SWalFileInfo* pLastFileInfo = taosArrayGet(pLogInfoArray, i-1); - if (pLastFileInfo->lastVer != pFileInfo->firstVer) { + taosArrayDestroy(pLogInfoArray); + pWal->writeCur = newSz - 1; + if (newSz > 0) { + pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; + + SWalFileInfo *pLastFileInfo = taosArrayGet(pWal->fileInfoSet, newSz-1); + char fnameStr[WAL_FILE_LEN]; + walBuildLogName(pWal, pLastFileInfo->firstVer, fnameStr); + struct stat statbuf; + stat(fnameStr, &statbuf); + + if (oldSz != newSz || pLastFileInfo->fileSize != statbuf.st_size) { + pLastFileInfo->fileSize = statbuf.st_size; + pWal->vers.lastVer = walScanLogGetLastVer(pWal); + ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer = pWal->vers.lastVer; + ASSERT(pWal->vers.lastVer != -1); + + int code = walSaveMeta(pWal); + if (code < 0) { + taosArrayDestroy(pLogInfoArray); + return -1; } } } -#endif - - // get last version of this file - // - // rebuild meta - taosArrayDestroy(pLogInfoArray); + //TODO: set fileSize and lastVer if necessary + return 0; } int walCheckAndRepairIdx(SWal* pWal) { - // iterate all idx files - // check first and last entry of each idx file valid + // TODO: iterate all log files + // if idx not found, scan log and write idx + // if found, check complete by first and last entry of each idx file + // if idx incomplete, binary search last valid entry, and then build other part return 0; } diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index aae4dc7074895eb5c1143058169d1e6362a8f7a4..0376ab70ad1bc0d0bb62d977773ad1566491a6d8 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -187,13 +187,19 @@ void *threadFunc(void *param) { int64_t curMs = 0; int64_t beginMs = taosGetTimestampMs(); pInfo->startMs = beginMs; - for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { - int64_t batch = (pInfo->tableEndIndex - t); - batch = MIN(batch, batchNum); + int64_t t = pInfo->tableBeginIndex; + for (; t <= pInfo->tableEndIndex;) { + //int64_t batch = (pInfo->tableEndIndex - t); + //batch = MIN(batch, batchNum); int32_t len = sprintf(qstr, "create table"); - for (int32_t i = 0; i < batch; ++i) { - len += sprintf(qstr + len, " t%" PRId64 " using %s tags(%" PRId64 ")", t + i, stbName, t + i); + for (int32_t i = 0; i < batchNum;) { + len += sprintf(qstr + len, " %s_t%" PRId64 " using %s tags(%" PRId64 ")", stbName, t, stbName, t); + t++; + i++; + if (t > pInfo->tableEndIndex) { + break; + } } int64_t startTs = taosGetTimestampUs(); @@ -212,11 +218,11 @@ void *threadFunc(void *param) { curMs = taosGetTimestampMs(); if (curMs - beginMs > 10000) { beginMs = curMs; + //printf("==== tableBeginIndex: %"PRId64", t: %"PRId64"\n", pInfo->tableBeginIndex, t); printCreateProgress(pInfo, t); } - t += (batch - 1); } - printCreateProgress(pInfo, pInfo->tableEndIndex); + printCreateProgress(pInfo, t); } if (insertData) {