diff --git a/2.0/src/query/src/queryMain.c b/2.0/src/query/src/queryMain.c index 3bf031ebd5b5afee8c6255be6dcd8e0b8193c653..e6d804cea375c4b4f2fc8e10e34fdf50e823483f 100644 --- a/2.0/src/query/src/queryMain.c +++ b/2.0/src/query/src/queryMain.c @@ -461,7 +461,7 @@ int32_t qKillQuery(qinfo_t qinfo) { } qDebug("QInfo:0x%"PRIx64" query killed", pQInfo->qId); - setQueryKilled(pQInfo); + setTaskKilled(pQInfo); // Wait for the query executing thread being stopped/ // Once the query is stopped, the owner of qHandle will be cleared immediately. @@ -634,7 +634,7 @@ int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCo return TSDB_CODE_QRY_INVALID_QHANDLE; } qWarn("QId:0x%"PRIx64" be killed(no memory commit).", pQInfo->qId); - setQueryKilled(pQInfo); + setTaskKilled(pQInfo); // wait query stop int32_t loop = 0; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1316847f485006f974431ae1e673c0761f41866b..210804143b78a93ed7871d1e1fa7d9267c6f965e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -107,7 +107,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_CLUSTER, TSDB_MGMT_TABLE_STREAMTABLES, TSDB_MGMT_TABLE_TP, - TSDB_MGMT_TABLE_FUNCTION, + TSDB_MGMT_TABLE_FUNC, TSDB_MGMT_TABLE_MAX, } EShowType; @@ -533,20 +533,21 @@ typedef struct { typedef struct { char name[TSDB_FUNC_NAME_LEN]; + int8_t igExists; int8_t funcType; int8_t scriptType; - int8_t align; int8_t outputType; int32_t outputLen; int32_t bufSize; - int64_t sigature; + int64_t signature; int32_t commentSize; int32_t codeSize; char pCont[]; } SCreateFuncReq; typedef struct { - char name[TSDB_FUNC_NAME_LEN]; + char name[TSDB_FUNC_NAME_LEN]; + int8_t igNotExists; } SDropFuncReq; typedef struct { @@ -556,13 +557,13 @@ typedef struct { typedef struct { char name[TSDB_FUNC_NAME_LEN]; + int8_t align; int8_t funcType; int8_t scriptType; - int8_t align; int8_t outputType; int32_t outputLen; int32_t bufSize; - int64_t sigature; + int64_t signature; int32_t commentSize; int32_t codeSize; char pCont[]; @@ -1247,6 +1248,12 @@ typedef struct { char data[]; } SVShowTablesFetchRsp; +typedef struct SMqCMGetSubEpReq { + int64_t consumerId; + int32_t epoch; + char cgroup[TSDB_CONSUMER_GROUP_LEN]; +} SMqCMGetSubEpReq; + #pragma pack(pop) static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) { @@ -1521,15 +1528,17 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { } typedef struct SMqSetCVgReq { - int32_t vgId; - int64_t oldConsumerId; - int64_t newConsumerId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CONSUMER_GROUP_LEN]; - char* sql; - char* logicalPlan; - char* physicalPlan; - SSubQueryMsg msg; + int32_t vgId; + int64_t oldConsumerId; + int64_t newConsumerId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cgroup[TSDB_CONSUMER_GROUP_LEN]; + char* sql; + char* logicalPlan; + char* physicalPlan; + uint32_t qmsgLen; + void* qmsg; + //SSubQueryMsg msg; } SMqSetCVgReq; static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* pMsg) { @@ -1538,7 +1547,7 @@ static FORCE_INLINE int32_t tEncodeSSubQueryMsg(void** buf, const SSubQueryMsg* tlen += taosEncodeFixedU64(buf, pMsg->queryId); tlen += taosEncodeFixedU64(buf, pMsg->taskId); tlen += taosEncodeFixedU32(buf, pMsg->contentLen); - tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen); + //tlen += taosEncodeBinary(buf, pMsg->msg, pMsg->contentLen); return tlen; } @@ -1547,7 +1556,7 @@ static FORCE_INLINE void* tDecodeSSubQueryMsg(void* buf, SSubQueryMsg* pMsg) { buf = taosDecodeFixedU64(buf, &pMsg->queryId); buf = taosDecodeFixedU64(buf, &pMsg->taskId); buf = taosDecodeFixedU32(buf, &pMsg->contentLen); - buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen); + //buf = taosDecodeBinaryTo(buf, pMsg->msg, pMsg->contentLen); return buf; } @@ -1561,7 +1570,9 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* tlen += taosEncodeString(buf, pReq->sql); tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan); - tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); + tlen += taosEncodeFixedU32(buf, pReq->qmsgLen); + tlen += taosEncodeString(buf, (char*)pReq->qmsg); + //tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); return tlen; } @@ -1574,15 +1585,18 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { buf = taosDecodeString(buf, &pReq->sql); buf = taosDecodeString(buf, &pReq->logicalPlan); buf = taosDecodeString(buf, &pReq->physicalPlan); - buf = tDecodeSSubQueryMsg(buf, &pReq->msg); + buf = taosDecodeFixedU32(buf, &pReq->qmsgLen); + buf = taosDecodeString(buf, (char**)&pReq->qmsg); + //buf = tDecodeSSubQueryMsg(buf, &pReq->msg); return buf; } typedef struct SMqSetCVgRsp { - int32_t vgId; - int64_t consumerId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cGroup[TSDB_CONSUMER_GROUP_LEN]; + SMsgHead header; + int32_t vgId; + int64_t consumerId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cGroup[TSDB_CONSUMER_GROUP_LEN]; } SMqSetCVgRsp; typedef struct SMqColData { @@ -1634,6 +1648,92 @@ typedef struct SMqConsumeReq { char topic[TSDB_TOPIC_FNAME_LEN]; } SMqConsumeReq; +typedef struct SMqSubVgEp { + int32_t vgId; + SEpSet epSet; +} SMqSubVgEp; + +typedef struct SMqSubTopicEp { + char topic[TSDB_TOPIC_FNAME_LEN]; + SArray* vgs; // SArray +} SMqSubTopicEp; + +typedef struct SMqCMGetSubEpRsp { + int64_t consumerId; + char cgroup[TSDB_CONSUMER_GROUP_LEN]; + SArray* topics; // SArray +} SMqCMGetSubEpRsp; + +static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI16(buf, pVgEp->vgId); + tlen += taosEncodeSEpSet(buf, &pVgEp->epSet); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { + buf = taosDecodeFixedI32(buf, &pVgEp->vgId); + buf = taosDecodeSEpSet(buf, &pVgEp->epSet); + return buf; +} + +static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pTopicEp->topic); + int32_t sz = taosArrayGetSize(pTopicEp->vgs); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i); + tlen += tEncodeSMqSubVgEp(buf, pVgEp); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { + buf = taosDecodeStringTo(buf, pTopicEp->topic); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); + if (pTopicEp->vgs == NULL) { + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp vgEp; + buf = tDecodeSMqSubVgEp(buf, &vgEp); + taosArrayPush(pTopicEp->vgs, &vgEp); + } + return buf; +} + +static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pRsp->consumerId); + tlen += taosEncodeString(buf, pRsp->cgroup); + int32_t sz = taosArrayGetSize(pRsp->topics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqSubTopicEp* pVgEp = (SMqSubTopicEp*)taosArrayGet(pRsp->topics, i); + tlen += tEncodeSMqSubTopicEp(buf, pVgEp); + } + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) { + buf = taosDecodeFixedI64(buf, &pRsp->consumerId); + buf = taosDecodeStringTo(buf, pRsp->cgroup); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pRsp->topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); + if (pRsp->topics == NULL) { + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqSubTopicEp topicEp; + buf = tDecodeSMqSubTopicEp(buf, &topicEp); + taosArrayPush(pRsp->topics, &topicEp); + } + return buf; +} #ifdef __cplusplus } diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 2ff4d58e0b77f60a51a62b759d6509b525d32bfd..e81dd798f668f9f37d73e5d4b4ed66bae08e61d2 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -119,9 +119,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_DB, "mnode-alter-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SYNC_DB, "mnode-sync-db", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_COMPACT_DB, "mnode-compact-db", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNCTION, "mnode-create-function", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_FUNCTION, "mnode-retrieve-function", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_FUNCTION, "mnode-drop-function", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_FUNC, "mnode-create-func", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_FUNC, "mnode-retrieve-func", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DROP_FUNC, "mnode-drop-func", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_STB, "mnode-create-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_STB, "mnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_STB, "mnode-drop-stb", NULL, NULL) @@ -140,6 +140,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) + TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-timer", SMqTmrMsg, SMqTmrMsg) // Requests handled by VNODE diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 457245e9a34ca763a1a129e260da90c2c73e12e6..d2e602a5d64f390580f644ab14bef6f5ddc553ba 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -32,7 +32,7 @@ struct SSubplan; * @param streamReadHandle * @return */ -qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg *pMsg, void* streamReadHandle); +qTaskInfo_t qCreateStreamExecTaskInfo(void *msg, void* streamReadHandle); int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input); @@ -45,7 +45,7 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input); * @param qId * @return */ -int32_t qCreateExecTask(void* readHandle, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); +int32_t qCreateExecTask(void* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); /** * The main task execution function, including query on both table and multiple tables, diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index ecfa344f8565911f822a1f7b257ba0556e7cbd5d..1d193888c9c8ba5bc6348bcb4d34a6d68e4adb29 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -74,6 +74,7 @@ void columnListCopy(SArray* dst, const SArray* src, uint64_t uid); void columnListDestroy(SArray* pColumnList); void dropAllExprInfo(SArray** pExprInfo, int32_t numOfLevel); +void dropOneLevelExprInfo(SArray* pExprInfo); typedef struct SSourceParam { SArray *pExprNodeList; //Array diff --git a/include/os/os.h b/include/os/os.h index 9112b4922f9e60109964cba5e1d34f2a021a886f..023d2b04705074abcec4c1dc927211bf63512112 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -51,6 +51,7 @@ extern "C" { #include #include +#include #include "osAtomic.h" #include "osDef.h" diff --git a/include/os/osDef.h b/include/os/osDef.h index 040c4bc7e72571435c32fe90cbdf82a93c9eb419..07d360a7c00a1c062e87d9512b91d86da203ce56 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -181,7 +181,8 @@ extern "C" { #endif #else // Windows - #define setThreadName(name) +// #define setThreadName(name) +#define setThreadName(name) do { prctl(PR_SET_NAME, (name)); } while (0) #endif #if defined(_WIN32) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 8c048690c021cce498ea73120e8c3bb565a84936..69bf085491101df1e131b7656489f8fb8feec1e8 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -239,6 +239,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x03C4) #define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x03C5) #define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x03C6) +#define TSDB_CODE_MND_INVALID_FUNC_RETRIEVE TAOS_DEF_ERROR_CODE(0, 0x03C7) // mnode-trans #define TSDB_CODE_MND_TRANS_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03D0) diff --git a/include/util/tdef.h b/include/util/tdef.h index 4c29d9963d82af99077b51eee6d15effb44b0c75..4d3ef06bd6bd11332bb3345543627aaa09e80603 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -173,6 +173,7 @@ do { \ #define TSDB_FUNC_BUF_SIZE 512 #define TSDB_FUNC_TYPE_SCALAR 1 #define TSDB_FUNC_TYPE_AGGREGATE 2 +#define TSDB_FUNC_MAX_RETRIEVE 1024 #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) diff --git a/include/util/tlist.h b/include/util/tlist.h index 06d0abc797befe3e9954e37513318207ec650090..134873a993f3721e860ec443da281e2443dd04a1 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -34,6 +34,7 @@ extern "C" { #define TD_SLIST_HEAD(sl) ((sl)->sl_head_) #define TD_SLIST_NELES(sl) ((sl)->sl_neles_) #define TD_SLIST_NODE_NEXT(sln) ((sln)->sl_next_) +#define TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) ((sln)->field.sl_next_) #define TD_SLIST_INIT(sl) \ do { \ @@ -48,12 +49,25 @@ extern "C" { TD_SLIST_NELES(sl) += 1; \ } while (0) +#define TD_SLIST_PUSH_WITH_FIELD(sl, sln, field) \ + do { \ + TD_SLIST_NODE_NEXT_WITH_FIELD(sln, field) = TD_SLIST_HEAD(sl); \ + TD_SLIST_HEAD(sl) = (sln); \ + TD_SLIST_NELES(sl) += 1; \ + } while (0) + #define TD_SLIST_POP(sl) \ do { \ TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT(TD_SLIST_HEAD(sl)); \ TD_SLIST_NELES(sl) -= 1; \ } while (0) +#define TD_SLIST_POP_WITH_FIELD(sl, field) \ + do { \ + TD_SLIST_HEAD(sl) = TD_SLIST_NODE_NEXT_WITH_FIELD(TD_SLIST_HEAD(sl), field); \ + TD_SLIST_NELES(sl) -= 1; \ + } while (0) + // Double linked list ================ #define TD_DLIST_NODE(TYPE) \ struct { \ @@ -70,6 +84,8 @@ extern "C" { #define TD_DLIST_NODE_PREV(dln) ((dln)->dl_prev_) #define TD_DLIST_NODE_NEXT(dln) ((dln)->dl_next_) +#define TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) ((dln)->field.dl_prev_) +#define TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) ((dln)->field.dl_next_) #define TD_DLIST_HEAD(dl) ((dl)->dl_head_) #define TD_DLIST_TAIL(dl) ((dl)->dl_tail_) #define TD_DLIST_NELES(dl) ((dl)->dl_neles_) @@ -94,6 +110,20 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) +#define TD_DLIST_APPEND_WITH_FIELD(dl, dln, field) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_TAIL(dl); \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_TAIL(dl), field) = (dln); \ + TD_DLIST_TAIL(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + #define TD_DLIST_PREPEND(dl, dln) \ do { \ if (TD_DLIST_HEAD(dl) == NULL) { \ @@ -108,6 +138,20 @@ extern "C" { TD_DLIST_NELES(dl) += 1; \ } while (0) +#define TD_DLIST_PREPEND_WITH_FIELD(dl, dln, field) \ + do { \ + if (TD_DLIST_HEAD(dl) == NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_HEAD(dl) = TD_DLIST_TAIL(dl) = (dln); \ + } else { \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = NULL; \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = TD_DLIST_HEAD(dl); \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_HEAD(dl), field) = (dln); \ + TD_DLIST_HEAD(dl) = (dln); \ + } \ + TD_DLIST_NELES(dl) += 1; \ + } while (0) + #define TD_DLIST_POP(dl, dln) \ do { \ if (TD_DLIST_HEAD(dl) == (dln)) { \ @@ -126,6 +170,26 @@ extern "C" { TD_DLIST_NODE_PREV(dln) = TD_DLIST_NODE_NEXT(dln) = NULL; \ } while (0) +#define TD_DLIST_POP_WITH_FIELD(dl, dln, field) \ + do { \ + if (TD_DLIST_HEAD(dl) == (dln)) { \ + TD_DLIST_HEAD(dl) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field); \ + } \ + if (TD_DLIST_TAIL(dl) == (dln)) { \ + TD_DLIST_TAIL(dl) = TD_DLIST_NODE_PREV_WITH_FIELD(dln, field); \ + } \ + if (TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) != NULL) { \ + TD_DLIST_NODE_NEXT_WITH_FIELD(TD_DLIST_NODE_PREV_WITH_FIELD(dln, field), field) = \ + TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field); \ + } \ + if (TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) != NULL) { \ + TD_DLIST_NODE_PREV_WITH_FIELD(TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field), field) = \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field); \ + } \ + TD_DLIST_NELES(dl) -= 1; \ + TD_DLIST_NODE_PREV_WITH_FIELD(dln, field) = TD_DLIST_NODE_NEXT_WITH_FIELD(dln, field) = NULL; \ + } while (0) + // General double linked list typedef enum { TD_LIST_FORWARD, TD_LIST_BACKWARD } TD_LIST_DIRECTION_T; diff --git a/include/util/tqueue.h b/include/util/tqueue.h index 63ba460d392c7cec868c7533613be1985d8a2f11..cfa5a65c2a14543e6690fcf0b886d77696857cb0 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -15,6 +15,7 @@ #ifndef _TD_UTIL_QUEUE_H #define _TD_UTIL_QUEUE_H +#include "os.h" #ifdef __cplusplus extern "C" { @@ -40,13 +41,13 @@ shall be used to set up the protection. typedef struct STaosQueue STaosQueue; typedef struct STaosQset STaosQset; typedef struct STaosQall STaosQall; -typedef void (*FProcessItem)(void *ahandle, void *pItem); -typedef void (*FProcessItems)(void *ahandle, STaosQall *qall, int32_t numOfItems); +typedef void (*FItem)(void *ahandle, void *pItem); +typedef void (*FItems)(void *ahandle, STaosQall *qall, int32_t numOfItems); STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); -void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsFp); -void *taosAllocateQitem(int32_t size); +void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp); +void * taosAllocateQitem(int32_t size); void taosFreeQitem(void *pItem); int32_t taosWriteQitem(STaosQueue *queue, void *pItem); int32_t taosReadQitem(STaosQueue *queue, void **ppItem); @@ -66,8 +67,11 @@ int32_t taosAddIntoQset(STaosQset *qset, STaosQueue *queue, void *ahandle); void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue); int32_t taosGetQueueNumber(STaosQset *qset); -int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FProcessItem *itemFp); -int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FProcessItems *itemsFp); +int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp); +int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp); + +int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp, int32_t threadId); +void taosResetQsetThread(STaosQset *qset, void *pItem); int32_t taosGetQueueItemsNumber(STaosQueue *queue); int32_t taosGetQsetItemsNumber(STaosQset *qset); diff --git a/include/util/tworker.h b/include/util/tworker.h index 27f03bd2b69e6d6add97cdc2e5ab05ad2fdd5eca..771c7c943359b08e664f3d7de45fe6a4ecf6ada3 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -15,57 +15,61 @@ #ifndef _TD_UTIL_WORKER_H #define _TD_UTIL_WORKER_H - #include "tqueue.h" #ifdef __cplusplus extern "C" { #endif -typedef struct SWorkerPool SWorkerPool; -typedef struct SMWorkerPool SMWorkerPool; +typedef struct SQWorkerPool SQWorkerPool; +typedef struct SWWorkerPool SWWorkerPool; -typedef struct SWorker { - int32_t id; // worker ID - pthread_t thread; // thread - SWorkerPool *pool; -} SWorker; +typedef struct SQWorker { + int32_t id; // worker ID + pthread_t thread; // thread + SQWorkerPool *pool; +} SQWorker, SFWorker; -typedef struct SWorkerPool { +typedef struct SQWorkerPool { int32_t max; // max number of workers int32_t min; // min number of workers int32_t num; // current number of workers - STaosQset *qset; - const char *name; - SWorker *workers; + STaosQset * qset; + const char * name; + SQWorker * workers; pthread_mutex_t mutex; -} SWorkerPool; +} SQWorkerPool, SFWorkerPool; -typedef struct SMWorker { +typedef struct SWWorker { int32_t id; // worker id pthread_t thread; // thread - STaosQall *qall; - STaosQset *qset; // queue set - SMWorkerPool *pool; -} SMWorker; + STaosQall * qall; + STaosQset * qset; // queue set + SWWorkerPool *pool; +} SWWorker; -typedef struct SMWorkerPool { +typedef struct SWWorkerPool { int32_t max; // max number of workers int32_t nextId; // from 0 to max-1, cyclic - const char *name; - SMWorker *workers; + const char * name; + SWWorker * workers; pthread_mutex_t mutex; -} SMWorkerPool; +} SWWorkerPool; + +int32_t tQWorkerInit(SQWorkerPool *pool); +void tQWorkerCleanup(SQWorkerPool *pool); +STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp); +void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue); -int32_t tWorkerInit(SWorkerPool *pool); -void tWorkerCleanup(SWorkerPool *pool); -STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp); -void tWorkerFreeQueue(SWorkerPool *pool, STaosQueue *queue); +int32_t tFWorkerInit(SFWorkerPool *pool); +void tFWorkerCleanup(SFWorkerPool *pool); +STaosQueue *tFWorkerAllocQueue(SFWorkerPool *pool, void *ahandle, FItem fp); +void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue); -int32_t tMWorkerInit(SMWorkerPool *pool); -void tMWorkerCleanup(SMWorkerPool *pool); -STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp); -void tMWorkerFreeQueue(SMWorkerPool *pool, STaosQueue *queue); +int32_t tWWorkerInit(SWWorkerPool *pool); +void tWWorkerCleanup(SWWorkerPool *pool); +STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp); +void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue); #ifdef __cplusplus } diff --git a/packaging/release.sh b/packaging/release.sh new file mode 100755 index 0000000000000000000000000000000000000000..8ea30223a6f0c7d1f30d610442634605ffebc5b3 --- /dev/null +++ b/packaging/release.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# +# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os + +set -e +#set -x + +# set parameters by default value +version="3.0.0.0" + +curr_dir=$(pwd) + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/..)" + +echo "=======================new version number: ${verNumber}======================================" + +build_time=$(date +"%F %R") + +echo "top_dir: ${top_dir}" + +cd ${top_dir} +git pull || : + +echo "curr_dir: ${curr_dir}" + +# 2. cmake executable file +compile_dir="${top_dir}/debug" +if [ -d ${compile_dir} ]; then + rm -rf ${compile_dir} +fi + +mkdir -p ${compile_dir} + +cd ${compile_dir} + +echo "compile_dir: ${compile_dir}" + +cmake .. +make -j32 + +release_dir="${top_dir}/release" +if [ -d ${release_dir} ]; then + rm -rf ${release_dir} +fi + +mkdir -p ${release_dir} +cd ${release_dir} + +install_dir="${release_dir}/TDengine-server-${version}" +mkdir -p ${install_dir} +mkdir -p ${install_dir}/lib + +bin_files="${compile_dir}/source/dnode/mgmt/daemon/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table" +cp ${bin_files} ${install_dir}/ && chmod a+x ${install_dir}/* || : + + +cp ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/snode/libsnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/bnode/libbnode.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/wal/libwal.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/scheduler/libscheduler.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/planner/libplanner.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/parser/libparser.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/qcom/libqcom.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/transport/libtransport.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/function/libfunction.so ${install_dir}/lib/ +cp ${compile_dir}/source/common/libcommon.so ${install_dir}/lib/ +cp ${compile_dir}/source/os/libos.so ${install_dir}/lib/ +cp ${compile_dir}/source/dnode/mnode/sdb/libsdb.so ${install_dir}/lib/ +cp ${compile_dir}/source/libs/catalog/libcatalog.so ${install_dir}/lib/ + +pkg_name=${install_dir}-Linux-x64 + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : + + diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ce0e5949557ec585240b52f146e8eb3fbd9b1000..1132cf63e4e6b05129658e27a5ac1ea855062d7d 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -218,6 +218,7 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag, if (pQueryNode->type == TSDB_SQL_SELECT) { setResSchemaInfo(&pRequest->body.resInfo, pSchema, numOfCols); + tfree(pSchema); pRequest->type = TDMT_VND_QUERY; } else { tfree(pSchema); @@ -262,7 +263,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, SArray* pNodeList) typedef struct SMqClientVg { // statistics - int64_t consumeCnt; + int64_t pollCnt; // offset int64_t committedOffset; int64_t currentOffset; @@ -313,10 +314,10 @@ tmq_conf_t* tmq_conf_new() { } int32_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { - if (strcmp(key, "group.id")) { + if (strcmp(key, "group.id") == 0) { strcpy(conf->groupId, value); } - if (strcmp(key, "client.id")) { + if (strcmp(key, "client.id") == 0) { strcpy(conf->clientId, value); } return 0; @@ -344,6 +345,7 @@ tmq_t* taos_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t err strcpy(pTmq->groupId, conf->groupId); pTmq->commit_cb = conf->commit_cb; pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1); + pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); return pTmq; } @@ -364,7 +366,7 @@ tmq_list_t* tmq_list_new() { int32_t tmq_list_append(tmq_list_t* ptr, char* src) { if (ptr->cnt >= ptr->tot-1) return -1; - ptr->elems[ptr->cnt] = src; + ptr->elems[ptr->cnt] = strdup(src); ptr->cnt++; return 0; } @@ -376,8 +378,23 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { int32_t sz = topic_list->cnt; tmq->clientTopics = taosArrayInit(sz, sizeof(void*)); for (int i = 0; i < sz; i++) { - char* topicName = strdup(topic_list->elems[i]); - taosArrayPush(tmq->clientTopics, &topicName); + char* topicName = topic_list->elems[i]; + + SName name = {0}; + char* dbName = getDbOfConnection(tmq->pTscObj); + tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); + tNameFromString(&name, topicName, T_NAME_TABLE); + + char* topicFname = calloc(1, TSDB_TOPIC_FNAME_LEN); + if (topicFname == NULL) { + + } + tNameExtractFullName(&name, topicFname); + tscDebug("subscribe topic: %s", topicFname); + taosArrayPush(tmq->clientTopics, &topicFname); + /*SMqClientTopic topic = {*/ + /*.*/ + /*};*/ } SCMSubscribeReq req; req.topicNum = taosArrayGetSize(tmq->clientTopics); @@ -395,13 +412,12 @@ TAOS_RES* tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tSerializeSCMSubscribeReq(&abuf, &req); /*printf("formatted: %s\n", dagStr);*/ - pRequest = createRequest(tmq->pTscObj, NULL, NULL, TSDB_SQL_SELECT); + pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_SUBSCRIBE); if (pRequest == NULL) { tscError("failed to malloc sqlObj"); } pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; - pRequest->type = TDMT_MND_CREATE_TOPIC; SMsgSendInfo* body = buildMsgInfoImpl(pRequest); SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -529,7 +545,7 @@ TAOS_RES *taos_create_topic(TAOS* taos, const char* topicName, const char* sql, SCMCreateTopicReq req = { .name = (char*) topicFname, - .igExists = 0, + .igExists = 1, .physicalPlan = (char*) pStr, .sql = (char*) sql, .logicalPlan = "no logic plan", @@ -575,6 +591,37 @@ struct tmq_message_t { SMqConsumeRsp rsp; }; +int32_t tmq_poll_cb_inner(void* param, const SDataBuf* pMsg, int32_t code) { + return 0; +} + +int32_t tmq_ask_ep_cb(void* param, const SDataBuf* pMsg, int32_t code) { + tmq_t* tmq = (tmq_t*)param; + SMqCMGetSubEpRsp rsp; + tDecodeSMqCMGetSubEpRsp(pMsg->pData, &rsp); + int32_t sz = taosArrayGetSize(rsp.topics); + // TODO: lock + tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); + for (int32_t i = 0; i < sz; i++) { + SMqClientTopic topic = {0}; + SMqSubTopicEp* pTopicEp = taosArrayGet(rsp.topics, i); + topic.topicName = strdup(pTopicEp->topic); + int32_t vgSz = taosArrayGetSize(pTopicEp->vgs); + topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); + for (int32_t j = 0; j < vgSz; j++) { + SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); + SMqClientVg clientVg = { + .vgId = pVgEp->vgId, + .epSet = pVgEp->epSet + }; + taosArrayPush(topic.vgs, &clientVg); + } + taosArrayPush(tmq->clientTopics, &topic); + } + // unlock + return 0; +} + tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { if (tmq->clientTopics == NULL || taosArrayGetSize(tmq->clientTopics) == 0) { return NULL; @@ -584,9 +631,38 @@ tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { req.reqType = 1; req.blockingTime = blocking_time; req.consumerId = tmq->consumerId; + tmq_message_t* tmq_message = NULL; strcpy(req.cgroup, tmq->groupId); - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); + if (taosArrayGetSize(tmq->clientTopics) == 0) { + int32_t tlen = sizeof(SMqCMGetSubEpReq); + SMqCMGetSubEpReq* buf = malloc(tlen); + if (buf == NULL) { + tscError("failed to malloc get subscribe ep buf"); + } + buf->consumerId = htobe64(buf->consumerId); + + pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_MND_GET_SUB_EP); + if (pRequest == NULL) { + tscError("failed to malloc subscribe ep request"); + } + + pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen }; + + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = tmq; + sendInfo->fp = tmq_ask_ep_cb; + + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + + tsem_wait(&pRequest->body.rspSem); + } + + SMqClientTopic* pTopic = taosArrayGetP(tmq->clientTopics, tmq->nextTopicIdx); tmq->nextTopicIdx = (tmq->nextTopicIdx + 1) % taosArrayGetSize(tmq->clientTopics); strcpy(req.topic, pTopic->topicName); int32_t nextVgIdx = pTopic->nextVgIdx; @@ -597,14 +673,17 @@ tmq_message_t* tmq_consume_poll(tmq_t* tmq, int64_t blocking_time) { pRequest->body.requestMsg = (SDataBuf){ .pData = &req, .len = sizeof(SMqConsumeReq) }; pRequest->type = TDMT_VND_CONSUME; - SMsgSendInfo* body = buildMsgInfoImpl(pRequest); + SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); + sendInfo->requestObjRefId = 0; + sendInfo->param = &tmq_message; + sendInfo->fp = tmq_poll_cb_inner; int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, body); + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); tsem_wait(&pRequest->body.rspSem); - return (tmq_message_t*)pRequest->body.resInfo.pData; + return tmq_message; /*tsem_wait(&pRequest->body.rspSem);*/ diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index c57aedee2e2e400d6afccfbc537baf645f14746d..a1adf58f6a485c878fb47629451e8b3dacb58170 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -61,473 +61,472 @@ TEST(testCase, connect_Test) { taos_close(pConn); } -TEST(testCase, create_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, create_user_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); -TEST(testCase, show_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show users"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop user abc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show databases"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_close(pConn); -} - -TEST(testCase, create_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); + //TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); + //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + //} - pRes = taos_query(pConn, "create database abc1 vgroups 4"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_close(pConn); -} - -TEST(testCase, create_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); - if (taos_errno(pRes) != 0) { - printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); - if (taos_errno(pRes) != 0) { - printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //taos_free_result(pRes); + //taos_close(pConn); +//} - taos_close(pConn); -} +//TEST(testCase, create_account_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); -TEST(testCase, drop_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); + //TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); + //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + //} - TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); - if (taos_errno(pRes) != 0) { - printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - } + //taos_free_result(pRes); + //taos_close(pConn); +//} - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); +//TEST(testCase, drop_account_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); + //TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); + //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + //} - pRes = taos_query(pConn, "drop dnode 4"); - if (taos_errno(pRes) != 0) { - printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - } + //taos_free_result(pRes); + //taos_close(pConn); +//} - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, show_user_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); -TEST(testCase, use_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); + //TAOS_RES* pRes = taos_query(pConn, "show users"); + //TAOS_ROW pRow = NULL; - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //int32_t numOfFields = taos_num_fields(pRes); - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); + //char str[512] = {0}; + //while ((pRow = taos_fetch_row(pRes)) != NULL) { + //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + //printf("%s\n", str); + //} - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); + //taos_free_result(pRes); + //taos_close(pConn); +//} - taos_close(pConn); -} +//TEST(testCase, drop_user_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - TEST(testCase, drop_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); + //TAOS_RES* pRes = taos_query(pConn, "drop user abc"); + //if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + //printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + //} - showDB(pConn); + //taos_free_result(pRes); + //taos_close(pConn); +//} - TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); +//TEST(testCase, show_db_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - showDB(pConn); + //TAOS_RES* pRes = taos_query(pConn, "show databases"); + //TAOS_ROW pRow = NULL; - pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("create to drop db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - taos_close(pConn); -} + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //int32_t numOfFields = taos_num_fields(pRes); -TEST(testCase, create_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); + //char str[512] = {0}; + //while ((pRow = taos_fetch_row(pRes)) != NULL) { + //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + //printf("%s\n", str); + //} - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //taos_close(pConn); +//} - pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("error in create stable, reason:%s\n", taos_errstr(pRes)); - } +//TEST(testCase, create_db_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); + //TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); + //if (taos_errno(pRes) != 0) { + //printf("error in create db, reason:%s\n", taos_errstr(pRes)); + //} - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - taos_free_result(pRes); + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //ASSERT_TRUE(pFields == NULL); - pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); - } + //int32_t numOfFields = taos_num_fields(pRes); + //ASSERT_EQ(numOfFields, 0); - pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - pRes = taos_query(pConn, "drop stable `123_$^)`"); - if (taos_errno(pRes) != 0) { - printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); - } + //taos_free_result(pRes); - taos_close(pConn); -} + //pRes = taos_query(pConn, "create database abc1 vgroups 4"); + //if (taos_errno(pRes) != 0) { + //printf("error in create db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_close(pConn); +//} -TEST(testCase, create_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +//TEST(testCase, create_dnode_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); + //TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); + //if (taos_errno(pRes) != 0) { + //printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); - pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); - ASSERT_EQ(taos_errno(pRes), 0); + //pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); - taos_free_result(pRes); + //taos_close(pConn); +//} - pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); - ASSERT_NE(taos_errno(pRes), 0); +//TEST(testCase, drop_dnode_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - taos_free_result(pRes); - taos_close(pConn); -} + //TAOS_RES* pRes = taos_query(pConn, "drop dnode 3"); + //if (taos_errno(pRes) != 0) { + //printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + //} -TEST(testCase, create_ctable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //ASSERT_TRUE(pFields == NULL); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //int32_t numOfFields = taos_num_fields(pRes); + //ASSERT_EQ(numOfFields, 0); - pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //pRes = taos_query(pConn, "drop dnode 4"); + //if (taos_errno(pRes) != 0) { + //printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); + //} - pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); - } + //taos_free_result(pRes); + //taos_close(pConn); +//} - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, use_db_test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); -TEST(testCase, show_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != nullptr); + //TAOS_RES* pRes = taos_query(pConn, "use abc1"); + //if (taos_errno(pRes) != 0) { + //printf("error in use db, reason:%s\n", taos_errstr(pRes)); + //} - TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); - if (taos_errno(pRes) != 0) { - printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //ASSERT_TRUE(pFields == NULL); - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); + //int32_t numOfFields = taos_num_fields(pRes); + //ASSERT_EQ(numOfFields, 0); - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } + //taos_close(pConn); +//} - taos_free_result(pRes); - taos_close(pConn); -} + //TEST(testCase, drop_db_test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); -TEST(testCase, show_vgroup_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); + //showDB(pConn); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); + //if (taos_errno(pRes) != 0) { + //printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); - pRes = taos_query(pConn, "show vgroups"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } + //showDB(pConn); - TAOS_ROW pRow = NULL; + //pRes = taos_query(pConn, "create database abc1"); + //if (taos_errno(pRes) != 0) { + //printf("create to drop db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); + //taos_close(pConn); +//} - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); +//TEST(testCase, create_stable_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); + + //TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2"); + //if (taos_errno(pRes) != 0) { + //printf("error in create db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "create table if not exists abc1.st1(ts timestamp, k int) tags(a int)"); + //if (taos_errno(pRes) != 0) { + //printf("error in create stable, reason:%s\n", taos_errstr(pRes)); + //} + + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //ASSERT_TRUE(pFields == NULL); + + //int32_t numOfFields = taos_num_fields(pRes); + //ASSERT_EQ(numOfFields, 0); + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "create stable if not exists abc1.`123_$^)` (ts timestamp, `abc` int) tags(a int)"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + //} + + //pRes = taos_query(pConn, "use abc1"); + //taos_free_result(pRes); + //pRes = taos_query(pConn, "drop stable `123_$^)`"); + //if (taos_errno(pRes) != 0) { + //printf("failed to drop super table 123_$^), reason:%s\n", taos_errstr(pRes)); + //} + + //taos_close(pConn); +//} - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } +//TEST(testCase, create_table_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - taos_free_result(pRes); - taos_close(pConn); -} + //TAOS_RES* pRes = taos_query(pConn, "use abc1"); + //taos_free_result(pRes); -TEST(testCase, create_multiple_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); + //pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k int)"); + //ASSERT_EQ(taos_errno(pRes), 0); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - taos_close(pConn); - return; - } + //taos_free_result(pRes); - taos_free_result(pRes); + //pRes = taos_query(pConn, "create table if not exists tm0(ts timestamp, k blob)"); + //ASSERT_NE(taos_errno(pRes), 0); - pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } + //taos_free_result(pRes); + //taos_close(pConn); +//} - taos_free_result(pRes); - pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } +//TEST(testCase, create_ctable_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); + + //TAOS_RES* pRes = taos_query(pConn, "use abc1"); + //if (taos_errno(pRes) != 0) { + //printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int ) tags(a int)"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create stable, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); + //} + + //taos_free_result(pRes); + //taos_close(pConn); +//} - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); +//TEST(testCase, show_stable_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != nullptr); + + //TAOS_RES* pRes = taos_query(pConn, "show abc1.stables"); + //if (taos_errno(pRes) != 0) { + //printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //ASSERT_TRUE(false); + //} + + //TAOS_ROW pRow = NULL; + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //int32_t numOfFields = taos_num_fields(pRes); + + //char str[512] = {0}; + //while ((pRow = taos_fetch_row(pRes)) != NULL) { + //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + //printf("%s\n", str); + //} + + //taos_free_result(pRes); + //taos_close(pConn); +//} - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } +//TEST(testCase, show_vgroup_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - taos_free_result(pRes); + //TAOS_RES* pRes = taos_query(pConn, "use abc1"); + //if (taos_errno(pRes) != 0) { + //printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); - for (int32_t i = 0; i < 20; ++i) { - char sql[512] = {0}; - snprintf(sql, tListLen(sql), - "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, - (i + 1) * 30, (i + 2) * 40); - TAOS_RES* pres = taos_query(pConn, sql); - if (taos_errno(pres) != 0) { - printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); - } - taos_free_result(pres); - } + //pRes = taos_query(pConn, "show vgroups"); + //if (taos_errno(pRes) != 0) { + //printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //ASSERT_TRUE(false); + //} - taos_close(pConn); -} + //TAOS_ROW pRow = NULL; -TEST(testCase, show_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //int32_t numOfFields = taos_num_fields(pRes); - TAOS_RES* pRes = taos_query(pConn, "show tables"); - ASSERT_NE(taos_errno(pRes), 0); + //char str[512] = {0}; + //while ((pRow = taos_fetch_row(pRes)) != NULL) { + //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + //printf("%s\n", str); + //} - if (taos_errno(pRes) != 0) { - printf("expected failed to show tables, reason:%s\n", taos_errstr(pRes)); - } + //taos_free_result(pRes); + //taos_close(pConn); +//} - taos_free_result(pRes); +//TEST(testCase, create_multiple_tables) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //ASSERT_NE(pConn, nullptr); + + //TAOS_RES* pRes = taos_query(pConn, "use abc1"); + //if (taos_errno(pRes) != 0) { + //printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //taos_close(pConn); + //return; + //} + + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //ASSERT_TRUE(false); + //} + + //taos_free_result(pRes); + //pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //ASSERT_TRUE(false); + //} + + //TAOS_ROW pRow = NULL; + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //int32_t numOfFields = taos_num_fields(pRes); + + //char str[512] = {0}; + //while ((pRow = taos_fetch_row(pRes)) != NULL) { + //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + //printf("%s\n", str); + //} + + //taos_free_result(pRes); + + //for (int32_t i = 0; i < 20; ++i) { + //char sql[512] = {0}; + //snprintf(sql, tListLen(sql), + //"create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, + //(i + 1) * 30, (i + 2) * 40); + //TAOS_RES* pres = taos_query(pConn, sql); + //if (taos_errno(pres) != 0) { + //printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); + //} + //taos_free_result(pres); + //} + + //taos_close(pConn); +//} - pRes = taos_query(pConn, "show abc1.tables"); - if (taos_errno(pRes) != 0) { - printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - } +//TEST(testCase, show_table_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); + //TAOS_RES* pRes = taos_query(pConn, "show tables"); + //if (taos_errno(pRes) != 0) { + //printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //} - int32_t count = 0; - char str[512] = {0}; + //taos_free_result(pRes); - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%d: %s\n", ++count, str); - } + //pRes = taos_query(pConn, "show abc1.tables"); + //if (taos_errno(pRes) != 0) { + //printf("failed to show tables, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //} - taos_free_result(pRes); - taos_close(pConn); -} + //TAOS_ROW pRow = NULL; + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //int32_t numOfFields = taos_num_fields(pRes); -TEST(testCase, drop_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != nullptr); + //int32_t count = 0; + //char str[512] = {0}; - TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); - if (taos_errno(pRes) != 0) { - printf("error in creating db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //while ((pRow = taos_fetch_row(pRes)) != NULL) { + //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + //printf("%d: %s\n", ++count, str); + //} - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in using db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //taos_free_result(pRes); + //taos_close(pConn); +//} - pRes = taos_query(pConn, "drop stable st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); - } +//TEST(testCase, drop_stable_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != nullptr); + + //TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1"); + //if (taos_errno(pRes) != 0) { + //printf("error in creating db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "use abc1"); + //if (taos_errno(pRes) != 0) { + //printf("error in using db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "drop stable st1"); + //if (taos_errno(pRes) != 0) { + //printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); + //} + + //taos_free_result(pRes); + //taos_close(pConn); +//} - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, generated_request_id_test) { + //SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); -TEST(testCase, generated_request_id_test) { - SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - - for (int32_t i = 0; i < 50000; ++i) { - uint64_t v = generateRequestId(); - void* result = taosHashGet(phash, &v, sizeof(v)); - if (result != nullptr) { - printf("0x%lx, index:%d\n", v, i); - } - assert(result == nullptr); - taosHashPut(phash, &v, sizeof(v), NULL, 0); - } + //for (int32_t i = 0; i < 50000; ++i) { + //uint64_t v = generateRequestId(); + //void* result = taosHashGet(phash, &v, sizeof(v)); + //if (result != nullptr) { + //printf("0x%lx, index:%d\n", v, i); + //} + //assert(result == nullptr); + //taosHashPut(phash, &v, sizeof(v), NULL, 0); + //} - taosHashCleanup(phash); -} + //taosHashCleanup(phash); +//} TEST(testCase, create_topic_Test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -537,6 +536,7 @@ TEST(testCase, create_topic_Test) { if (taos_errno(pRes) != 0) { printf("error in use db, reason:%s\n", taos_errstr(pRes)); } + taos_free_result(pRes); TAOS_FIELD* pFields = taos_fetch_fields(pRes); ASSERT_TRUE(pFields == nullptr); @@ -552,74 +552,123 @@ TEST(testCase, create_topic_Test) { taos_close(pConn); } -TEST(testCase, insert_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, projection_query_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); +//TEST(testCase, insert_test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //ASSERT_NE(pConn, nullptr); - pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //TAOS_RES* pRes = taos_query(pConn, "use abc1"); + //taos_free_result(pRes); - pRes = taos_query(pConn, "create table tu using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); + //pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //ASSERT_TRUE(false); + //} - for(int32_t i = 0; i < 100000; ++i) { - char sql[512] = {0}; - sprintf(sql, "insert into tu values(now+%da, %d)", i, i); - TAOS_RES* p = taos_query(pConn, sql); - if (taos_errno(p) != 0) { - printf("failed to insert data, reason:%s\n", taos_errstr(p)); - } + //taos_free_result(pRes); + //taos_close(pConn); +//} - taos_free_result(p); - } +//TEST(testCase, tmq_subscribe_Test) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //assert(pConn != NULL); + + //TAOS_RES* pRes = taos_query(pConn, "use abc1"); + //if (taos_errno(pRes) != 0) { + //printf("error in use db, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); + + //tmq_conf_t* conf = tmq_conf_new(); + //tmq_conf_set(conf, "group.id", "tg1"); + //tmq_t* tmq = taos_consumer_new(pConn, conf, NULL, 0); + + //tmq_list_t* topic_list = tmq_list_new(); + //tmq_list_append(topic_list, "test_topic_1"); + //tmq_subscribe(tmq, topic_list); + + //while (1) { + //tmq_message_t* msg = tmq_consume_poll(tmq, 0); + //printf("get msg\n"); + //if (msg == NULL) break; + //} +//} - pRes = taos_query(pConn, "select * from tu"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } +TEST(testCase, tmq_consume_Test) { +} - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); +TEST(testCase, tmq_commit_TEST) { +} - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } +//TEST(testCase, insert_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into t_2 values(now, 1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} - taos_free_result(pRes); - taos_close(pConn); -} +//TEST(testCase, projection_query_tables) { + //TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + //ASSERT_NE(pConn, nullptr); + + //TAOS_RES* pRes = taos_query(pConn, "use abc1"); + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); + + //pRes = taos_query(pConn, "create table tu using st1 tags(1)"); + //if (taos_errno(pRes) != 0) { + //printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + //} + //taos_free_result(pRes); + + //for(int32_t i = 0; i < 100000; ++i) { + //char sql[512] = {0}; + //sprintf(sql, "insert into tu values(now+%da, %d)", i, i); + //TAOS_RES* p = taos_query(pConn, sql); + //if (taos_errno(p) != 0) { + //printf("failed to insert data, reason:%s\n", taos_errstr(p)); + //} + + //taos_free_result(p); + //} + + //pRes = taos_query(pConn, "select * from tu"); + //if (taos_errno(pRes) != 0) { + //printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + //taos_free_result(pRes); + //ASSERT_TRUE(false); + //} + + //TAOS_ROW pRow = NULL; + //TAOS_FIELD* pFields = taos_fetch_fields(pRes); + //int32_t numOfFields = taos_num_fields(pRes); + + //char str[512] = {0}; + //while ((pRow = taos_fetch_row(pRes)) != NULL) { + //int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + //printf("%s\n", str); + //} + + //taos_free_result(pRes); + //taos_close(pConn); +//} //TEST(testCase, projection_query_stables) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index 7ab3f46fdbe36c0dbd7c85b079c45ceb68825ee0..4214dca11d930c93240fccc58f58882e1c3c47b3 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -31,8 +31,8 @@ typedef struct { SDnode *pDnode; STaosQueue *queue; union { - SWorkerPool pool; - SMWorkerPool mpool; + SQWorkerPool pool; + SWWorkerPool mpool; }; } SDnodeWorker; @@ -109,10 +109,10 @@ typedef struct { int32_t openVnodes; int32_t totalVnodes; SRWLatch latch; - SWorkerPool queryPool; - SWorkerPool fetchPool; - SMWorkerPool syncPool; - SMWorkerPool writePool; + SQWorkerPool queryPool; + SFWorkerPool fetchPool; + SWWorkerPool syncPool; + SWWorkerPool writePool; } SVnodesMgmt; typedef struct { diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 507f9a2644489e76e92ac45b3e2a1bc81d7790f5..b127fb1d64dc08ac015b54a4f0eb75ebd312b2b3 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -487,6 +487,8 @@ static void *dnodeThreadRoutine(void *param) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; int32_t ms = pDnode->cfg.statusInterval * 1000; + setThreadName("dnode-hb"); + while (true) { pthread_testcancel(); taosMsleep(ms); diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 17aa9fa327fa0c78662e9ecc571ad1d020b7728e..275b09fc0fad0efa45f53cb38c70046a3b7b0ab2 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -25,9 +25,9 @@ #include "dndMnode.h" #include "dndVnodes.h" -#define INTERNAL_USER "_internal" +#define INTERNAL_USER "_dnd" #define INTERNAL_CKEY "_key" -#define INTERNAL_SECRET "_secret" +#define INTERNAL_SECRET "_pwd" static void dndInitMsgFp(STransMgmt *pMgmt) { // Requests handled by DNODE @@ -90,9 +90,9 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SYNC_DB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_COMPACT_DB)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNCTION)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNCTION)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNCTION)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_FUNC)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_FUNC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_STB)] = dndProcessMnodeWriteMsg; @@ -112,11 +112,15 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_MND_CREATE_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_ALTER_TOPIC)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_MND_DROP_TOPIC)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_SUBSCRIBE)] = dndProcessMnodeWriteMsg; + /*pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBSCRIBE_RSP)] = dndProcessMnodeWriteMsg;*/ + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN_RSP)] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_MND_GET_SUB_EP)] = dndProcessMnodeReadMsg; // Requests handled by VNODE pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SUBMIT)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY)] = dndProcessVnodeQueryMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_QUERY_CONTINUE)] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_FETCH)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_ALTER_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_UPDATE_TAG_VAL)] = dndProcessVnodeWriteMsg; @@ -142,6 +146,8 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_VND_DROP_TABLE)] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES)] = dndProcessVnodeFetchMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_VND_SHOW_TABLES_FETCH)] = dndProcessVnodeFetchMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CONN)] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TMSG_INDEX(TDMT_VND_MQ_SET_CUR)] = dndProcessVnodeFetchMsg; } static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { @@ -152,17 +158,18 @@ static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { if (dndGetStat(pDnode) == DND_STAT_STOPPED) { if (pRsp == NULL || pRsp->pCont == NULL) return; - dTrace("RPC %p, rsp:%s is ignored since dnode is stopping", pRsp->handle, TMSG_INFO(msgType)); + dTrace("RPC %p, rsp:%s ignored since dnode exiting, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle); rpcFreeCont(pRsp->pCont); return; } DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)]; if (fp != NULL) { - dTrace("RPC %p, rsp:%s will be processed, code:0x%x", pRsp->handle, TMSG_INFO(msgType), pRsp->code & 0XFFFF); + dTrace("RPC %p, rsp:%s will be processed, code:0x%x app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->code & 0XFFFF, + pRsp->ahandle); (*fp)(pDnode, pRsp, pEpSet); } else { - dError("RPC %p, rsp:%s not processed", pRsp->handle, TMSG_INFO(msgType)); + dError("RPC %p, rsp:%s not processed, app:%p", pRsp->handle, TMSG_INFO(msgType), pRsp->ahandle); rpcFreeCont(pRsp->pCont); } } @@ -189,7 +196,7 @@ static int32_t dndInitClient(SDnode *pDnode) { pMgmt->clientRpc = rpcOpen(&rpcInit); if (pMgmt->clientRpc == NULL) { - dError("failed to init rpc client"); + dError("failed to init dnode rpc client"); return -1; } @@ -212,40 +219,39 @@ static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { tmsg_t msgType = pReq->msgType; if (msgType == TDMT_DND_NETWORK_TEST) { - dTrace("RPC %p, network test req, app:%p will be processed, code:0x%x", pReq->handle, pReq->ahandle, pReq->code); + dTrace("RPC %p, network test req will be processed, app:%p", pReq->handle, pReq->ahandle); dndProcessStartupReq(pDnode, pReq); return; } if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - dError("RPC %p, req:%s app:%p is ignored since dnode exiting", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_OFFLINE}; + dError("RPC %p, req:%s ignored since dnode exiting, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_OFFLINE, .ahandle = pReq->ahandle}; rpcSendResponse(&rspMsg); rpcFreeCont(pReq->pCont); return; } else if (dndGetStat(pDnode) != DND_STAT_RUNNING) { - dError("RPC %p, req:%s app:%p is ignored since dnode not running", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY}; + dError("RPC %p, req:%s ignored since dnode not running, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_APP_NOT_READY, .ahandle = pReq->ahandle}; rpcSendResponse(&rspMsg); rpcFreeCont(pReq->pCont); return; } if (pReq->pCont == NULL) { - dTrace("RPC %p, req:%s app:%p not processed since content is null", pReq->handle, TMSG_INFO(msgType), - pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN}; + dTrace("RPC %p, req:%s not processed since its empty, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_DND_INVALID_MSG_LEN, .ahandle = pReq->ahandle}; rpcSendResponse(&rspMsg); return; } DndMsgFp fp = pMgmt->msgFp[TMSG_INDEX(msgType)]; if (fp != NULL) { - dTrace("RPC %p, req:%s app:%p will be processed", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); + dTrace("RPC %p, req:%s will be processed, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); (*fp)(pDnode, pReq, pEpSet); } else { - dError("RPC %p, req:%s app:%p is not processed since no handle", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); - SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED}; + dError("RPC %p, req:%s not processed since no handle, app:%p", pReq->handle, TMSG_INFO(msgType), pReq->ahandle); + SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; rpcSendResponse(&rspMsg); rpcFreeCont(pReq->pCont); } @@ -285,24 +291,24 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char SDnode *pDnode = parent; if (dndAuthInternalReq(parent, user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from internal mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); return 0; } if (dndGetUserAuthFromMnode(pDnode, user, spi, encrypt, secret, ckey) == 0) { - dTrace("user:%s, get auth from internal mnode, spi:%d encrypt:%d", user, *spi, *encrypt); + dTrace("user:%s, get auth from mnode, spi:%d encrypt:%d", user, *spi, *encrypt); return 0; } if (terrno != TSDB_CODE_APP_NOT_READY) { - dTrace("failed to get user auth from internal mnode since %s", terrstr()); + dTrace("failed to get user auth from mnode since %s", terrstr()); return -1; } SAuthReq *pReq = rpcMallocCont(sizeof(SAuthReq)); tstrncpy(pReq->user, user, TSDB_USER_LEN); - SRpcMsg rpcMsg = {.pCont = pReq, .contLen = sizeof(SAuthReq), .msgType = TDMT_MND_AUTH}; + SRpcMsg rpcMsg = {.pCont = pReq, .contLen = sizeof(SAuthReq), .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; SRpcMsg rpcRsp = {0}; dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, pReq->spi, pReq->encrypt); dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); @@ -346,7 +352,7 @@ static int32_t dndInitServer(SDnode *pDnode) { pMgmt->serverRpc = rpcOpen(&rpcInit); if (pMgmt->serverRpc == NULL) { - dError("failed to init rpc server"); + dError("failed to init dnode rpc server"); return -1; } diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index df6cdc10cfdf926174f14b079b1297233bce9238..dcb73d13c78c1a058fdbc13b3facd64b09fa1f3a 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -253,7 +253,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ } for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON *vnode = cJSON_GetArrayItem(vnodes, i); + cJSON * vnode = cJSON_GetArrayItem(vnodes, i); SWrapperCfg *pCfg = &pCfgs[i]; cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); @@ -382,7 +382,7 @@ static void *dnodeOpenVnodeFunc(void *param) { dndReportStartup(pDnode, "open-vnodes", stepDesc); SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId}; - SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); + SVnode * pImpl = vnodeOpen(pCfg->path, &cfg); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->failed++; @@ -737,15 +737,9 @@ int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { return 0; } -static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { - SRpcMsg *pRsp = NULL; - vnodeProcessQueryReq(pVnode->pImpl, pMsg, &pRsp); -} +static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessQueryMsg(pVnode->pImpl, pMsg); } -static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { - SRpcMsg *pRsp = NULL; - vnodeProcessFetchReq(pVnode->pImpl, pMsg, &pRsp); -} +static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SRpcMsg *pMsg) { vnodeProcessFetchMsg(pVnode->pImpl, pMsg); } static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *)); @@ -916,27 +910,27 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) { int32_t maxWriteThreads = TMAX(pDnode->env.numOfCores, 1); int32_t maxSyncThreads = TMAX(pDnode->env.numOfCores / 2, 1); - SWorkerPool *pPool = &pMgmt->queryPool; - pPool->name = "vnode-query"; - pPool->min = minQueryThreads; - pPool->max = maxQueryThreads; - if (tWorkerInit(pPool) != 0) return -1; + SQWorkerPool *pQPool = &pMgmt->queryPool; + pQPool->name = "vnode-query"; + pQPool->min = minQueryThreads; + pQPool->max = maxQueryThreads; + if (tQWorkerInit(pQPool) != 0) return -1; - pPool = &pMgmt->fetchPool; - pPool->name = "vnode-fetch"; - pPool->min = minFetchThreads; - pPool->max = maxFetchThreads; - if (tWorkerInit(pPool) != 0) return -1; + SFWorkerPool *pFPool = &pMgmt->fetchPool; + pFPool->name = "vnode-fetch"; + pFPool->min = minFetchThreads; + pFPool->max = maxFetchThreads; + if (tFWorkerInit(pFPool) != 0) return -1; - SMWorkerPool *pMPool = &pMgmt->writePool; - pMPool->name = "vnode-write"; - pMPool->max = maxWriteThreads; - if (tMWorkerInit(pMPool) != 0) return -1; + SWWorkerPool *pWPool = &pMgmt->writePool; + pWPool->name = "vnode-write"; + pWPool->max = maxWriteThreads; + if (tWWorkerInit(pWPool) != 0) return -1; - pMPool = &pMgmt->syncPool; - pMPool->name = "vnode-sync"; - pMPool->max = maxSyncThreads; - if (tMWorkerInit(pMPool) != 0) return -1; + pWPool = &pMgmt->syncPool; + pWPool->name = "vnode-sync"; + pWPool->max = maxSyncThreads; + if (tWWorkerInit(pWPool) != 0) return -1; dDebug("vnode workers is initialized"); return 0; @@ -944,21 +938,21 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) { static void dndCleanupVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tWorkerCleanup(&pMgmt->fetchPool); - tWorkerCleanup(&pMgmt->queryPool); - tMWorkerCleanup(&pMgmt->writePool); - tMWorkerCleanup(&pMgmt->syncPool); + tFWorkerCleanup(&pMgmt->fetchPool); + tQWorkerCleanup(&pMgmt->queryPool); + tWWorkerCleanup(&pMgmt->writePool); + tWWorkerCleanup(&pMgmt->syncPool); dDebug("vnode workers is closed"); } static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - pVnode->pWriteQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeWriteQueue); - pVnode->pApplyQ = tMWorkerAllocQueue(&pMgmt->writePool, pVnode, (FProcessItems)dndProcessVnodeApplyQueue); - pVnode->pSyncQ = tMWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FProcessItems)dndProcessVnodeSyncQueue); - pVnode->pFetchQ = tWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FProcessItem)dndProcessVnodeFetchQueue); - pVnode->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FProcessItem)dndProcessVnodeQueryQueue); + pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeWriteQueue); + pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)dndProcessVnodeApplyQueue); + pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)dndProcessVnodeSyncQueue); + pVnode->pFetchQ = tFWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)dndProcessVnodeFetchQueue); + pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)dndProcessVnodeQueryQueue); if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || pVnode->pQueryQ == NULL) { @@ -971,11 +965,11 @@ static int32_t dndAllocVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { static void dndFreeVnodeQueue(SDnode *pDnode, SVnodeObj *pVnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - tWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); - tWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); - tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); - tMWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); - tMWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); + tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); + tFWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); + tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); + tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); + tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); pVnode->pWriteQ = NULL; pVnode->pApplyQ = NULL; pVnode->pSyncQ = NULL; diff --git a/source/dnode/mgmt/impl/src/dndWorker.c b/source/dnode/mgmt/impl/src/dndWorker.c index e0db262f89b9cc3181174b0c3341757f23b313bb..5ccf6640c04789911932a1aabd2717f7efd3762a 100644 --- a/source/dnode/mgmt/impl/src/dndWorker.c +++ b/source/dnode/mgmt/impl/src/dndWorker.c @@ -31,28 +31,28 @@ int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, c pWorker->pDnode = pDnode; if (pWorker->type == DND_WORKER_SINGLE) { - SWorkerPool *pPool = &pWorker->pool; + SQWorkerPool *pPool = &pWorker->pool; pPool->name = name; pPool->min = minNum; pPool->max = maxNum; - if (tWorkerInit(pPool) != 0) { + if (tQWorkerInit(pPool) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pWorker->queue = tWorkerAllocQueue(pPool, pDnode, (FProcessItem)queueFp); + pWorker->queue = tQWorkerAllocQueue(pPool, pDnode, (FItem)queueFp); if (pWorker->queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } } else if (pWorker->type == DND_WORKER_MULTI) { - SMWorkerPool *pPool = &pWorker->mpool; + SWWorkerPool *pPool = &pWorker->mpool; pPool->name = name; pPool->max = maxNum; - if (tMWorkerInit(pPool) != 0) { + if (tWWorkerInit(pPool) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pWorker->queue = tMWorkerAllocQueue(pPool, pDnode, (FProcessItems)queueFp); + pWorker->queue = tWWorkerAllocQueue(pPool, pDnode, (FItems)queueFp); if (pWorker->queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -70,11 +70,11 @@ void dndCleanupWorker(SDnodeWorker *pWorker) { } if (pWorker->type == DND_WORKER_SINGLE) { - tWorkerCleanup(&pWorker->pool); - tWorkerFreeQueue(&pWorker->pool, pWorker->queue); + tQWorkerCleanup(&pWorker->pool); + tQWorkerFreeQueue(&pWorker->pool, pWorker->queue); } else if (pWorker->type == DND_WORKER_MULTI) { - tMWorkerCleanup(&pWorker->mpool); - tMWorkerFreeQueue(&pWorker->mpool, pWorker->queue); + tWWorkerCleanup(&pWorker->mpool); + tWWorkerFreeQueue(&pWorker->mpool, pWorker->queue); } else { } } diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 9f724faeb9a8d985a6c52ce1ba396634927b0ed9..955f62610a940731b95666a2d4b5f6197af5d9ca 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -96,6 +96,15 @@ class Testbase { #define CheckBinary(val, len) \ { EXPECT_STREQ(test.GetShowBinary(len), val); } +#define CheckBinaryByte(b, len) \ + { \ + char* bytes = (char*)calloc(1, len); \ + for (int32_t i = 0; i < len - 1; ++i) { \ + bytes[i] = b; \ + } \ + EXPECT_STREQ(test.GetShowBinary(len), bytes); \ + } + #define CheckInt8(val) \ { EXPECT_EQ(test.GetShowInt8(), val); } diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 9d1dd084ee27cc44febf1c19a26bc20cb4a35c22..42b2de01cc4ff38d53b310741c1ccf9677f540ae 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -25,7 +25,7 @@ extern "C" { int32_t mndInitConsumer(SMnode *pMnode); void mndCleanupConsumer(SMnode *pMnode); -SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId); +SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId); void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer); SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 78f371133cceeca9963e4b7f755ccb1e61a8a99d..02ce3a159176a2e35806281a2e733af214f1b917 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -314,7 +314,7 @@ typedef struct { int8_t outputType; int32_t outputLen; int32_t bufSize; - int64_t sigature; + int64_t signature; int32_t commentSize; int32_t codeSize; char* pComment; @@ -363,8 +363,9 @@ typedef struct SMqConsumerEp { int64_t consumerId; // -1 for unassigned int64_t lastConsumerHbTs; int64_t lastVgHbTs; - int32_t execLen; - SSubQueryMsg qExec; + uint32_t qmsgLen; + char* qmsg; + //SSubQueryMsg qExec; } SMqConsumerEp; static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsumerEp) { @@ -373,7 +374,9 @@ static FORCE_INLINE int32_t tEncodeSMqConsumerEp(void** buf, SMqConsumerEp* pCon tlen += taosEncodeFixedI32(buf, pConsumerEp->status); tlen += taosEncodeSEpSet(buf, &pConsumerEp->epSet); tlen += taosEncodeFixedI64(buf, pConsumerEp->consumerId); - tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec); + //tlen += tEncodeSSubQueryMsg(buf, &pConsumerEp->qExec); + tlen += taosEncodeFixedU32(buf, pConsumerEp->qmsgLen); + tlen += taosEncodeBinary(buf, pConsumerEp->qmsg, pConsumerEp->qmsgLen); return tlen; } @@ -382,8 +385,9 @@ static FORCE_INLINE void* tDecodeSMqConsumerEp(void** buf, SMqConsumerEp* pConsu buf = taosDecodeFixedI32(buf, &pConsumerEp->status); buf = taosDecodeSEpSet(buf, &pConsumerEp->epSet); buf = taosDecodeFixedI64(buf, &pConsumerEp->consumerId); - buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec); - pConsumerEp->execLen = sizeof(SSubQueryMsg) + pConsumerEp->qExec.contentLen; + //buf = tDecodeSSubQueryMsg(buf, &pConsumerEp->qExec); + buf = taosDecodeFixedU32(buf, &pConsumerEp->qmsgLen); + buf = taosDecodeBinary(buf, (void**)&pConsumerEp->qmsg, pConsumerEp->qmsgLen); return buf; } @@ -402,11 +406,12 @@ typedef struct SMqSubscribeObj { static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { SMqSubscribeObj* pSub = malloc(sizeof(SMqSubscribeObj)); - pSub->key[0] = 0; - pSub->epoch = 0; if (pSub == NULL) { return NULL; } + pSub->key[0] = 0; + pSub->epoch = 0; + pSub->availConsumer = taosArrayInit(0, sizeof(int64_t)); if (pSub->availConsumer == NULL) { free(pSub); @@ -433,7 +438,7 @@ static FORCE_INLINE SMqSubscribeObj* tNewSubscribeObj() { free(pSub); return NULL; } - return NULL; + return pSub; } static FORCE_INLINE int32_t tEncodeSubscribeObj(void** buf, const SMqSubscribeObj* pSub) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 5cdd8e77bd627464d462df6e3ff000280df7ea3d..e642b578fa6e0a1359a7a2ab1355b5602c192123 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -30,24 +30,23 @@ #define MND_CONSUMER_VER_NUMBER 1 #define MND_CONSUMER_RESERVE_SIZE 64 -static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); -static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); -static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); -static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg); -static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); -static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); +static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); +static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); +static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer); +static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg); +static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); int32_t mndInitConsumer(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_CONSUMER, - .keyType = SDB_KEY_BINARY, + .keyType = SDB_KEY_INT64, .encodeFp = (SdbEncodeFp)mndConsumerActionEncode, .decodeFp = (SdbDecodeFp)mndConsumerActionDecode, .insertFp = (SdbInsertFp)mndConsumerActionInsert, .updateFp = (SdbUpdateFp)mndConsumerActionUpdate, .deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; - return sdbSetTable(pMnode->pSdb, table); } @@ -56,46 +55,20 @@ void mndCleanupConsumer(SMnode *pMnode) {} SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t tlen = tEncodeSMqConsumerObj(NULL, pConsumer); - SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, tlen); + int32_t size = sizeof(int32_t) + tlen + MND_CONSUMER_RESERVE_SIZE; + + SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); if (pRaw == NULL) goto CM_ENCODE_OVER; - void* buf = malloc(tlen); + void *buf = malloc(tlen); if (buf == NULL) goto CM_ENCODE_OVER; - void* abuf = buf; + void *abuf = buf; tEncodeSMqConsumerObj(&abuf, pConsumer); int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, tlen, CM_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, buf, tlen, CM_ENCODE_OVER); - -#if 0 - int32_t topicNum = taosArrayGetSize(pConsumer->topics); - SDB_SET_INT64(pRaw, dataPos, pConsumer->consumerId, CM_ENCODE_OVER); - int32_t len = strlen(pConsumer->cgroup); - SDB_SET_INT32(pRaw, dataPos, len, CM_ENCODE_OVER); - SDB_SET_BINARY(pRaw, dataPos, pConsumer->cgroup, len, CM_ENCODE_OVER); - SDB_SET_INT32(pRaw, dataPos, topicNum, CM_ENCODE_OVER); - for (int i = 0; i < topicNum; i++) { - int32_t len; - SMqConsumerTopic *pConsumerTopic = taosArrayGet(pConsumer->topics, i); - len = strlen(pConsumerTopic->name); - SDB_SET_INT32(pRaw, dataPos, len, CM_ENCODE_OVER); - SDB_SET_BINARY(pRaw, dataPos, pConsumerTopic->name, len, CM_ENCODE_OVER); - int vgSize; - if (pConsumerTopic->vgroups == NULL) { - vgSize = 0; - } else { - vgSize = listNEles(pConsumerTopic->vgroups); - } - SDB_SET_INT32(pRaw, dataPos, vgSize, CM_ENCODE_OVER); - for (int j = 0; j < vgSize; j++) { - // SList* head; - /*SDB_SET_INT64(pRaw, dataPos, 0[> change to list item <]);*/ - } - } -#endif - SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_ENCODE_OVER); SDB_SET_DATALEN(pRaw, dataPos, CM_ENCODE_OVER); @@ -116,53 +89,35 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CONSUME_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER; if (sver != MND_CONSUMER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto CONSUME_DECODE_OVER; + goto CM_DECODE_OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SMqConsumerObj)); - if (pRow == NULL) goto CONSUME_DECODE_OVER; + if (pRow == NULL) goto CM_DECODE_OVER; SMqConsumerObj *pConsumer = sdbGetRowObj(pRow); - if (pConsumer == NULL) goto CONSUME_DECODE_OVER; + if (pConsumer == NULL) goto CM_DECODE_OVER; int32_t dataPos = 0; int32_t len; - SDB_GET_INT32(pRaw, dataPos, &len, CONSUME_DECODE_OVER); - void* buf = malloc(len); - if (buf == NULL) goto CONSUME_DECODE_OVER; - - SDB_GET_BINARY(pRaw, dataPos, buf, len, CONSUME_DECODE_OVER); - - tDecodeSMqConsumerObj(buf, pConsumer); - - SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CONSUME_DECODE_OVER); + SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER); + void *buf = malloc(len); + if (buf == NULL) goto CM_DECODE_OVER; + SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); + SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); + + if (tDecodeSMqConsumerObj(buf, pConsumer) == NULL) { + goto CM_DECODE_OVER; + } terrno = TSDB_CODE_SUCCESS; -#if 0 - SDB_GET_INT32(pRaw, dataPos, &topicNum, CONSUME_DECODE_OVER); - for (int i = 0; i < topicNum; i++) { - int32_t topicLen; - SMqConsumerTopic *pConsumerTopic = malloc(sizeof(SMqConsumerTopic)); - if (pConsumerTopic == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - // TODO - return NULL; - } - /*pConsumerTopic->vgroups = taosArrayInit(topicNum, sizeof(SMqConsumerTopic));*/ - SDB_GET_INT32(pRaw, dataPos, &topicLen, CONSUME_DECODE_OVER); - SDB_GET_BINARY(pRaw, dataPos, pConsumerTopic->name, topicLen, CONSUME_DECODE_OVER); - int32_t vgSize; - SDB_GET_INT32(pRaw, dataPos, &vgSize, CONSUME_DECODE_OVER); - } -#endif - -CONSUME_DECODE_OVER: - if (terrno != 0) { +CM_DECODE_OVER: + if (terrno != TSDB_CODE_SUCCESS) { mError("consumer:%ld, failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr()); tfree(pRow); return NULL; @@ -191,7 +146,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, return 0; } -SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId) { +SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int64_t consumerId) { SSdb *pSdb = pMnode->pSdb; SMqConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId); if (pConsumer == NULL) { diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index d406247bc1e345a5f767dd7d0651b6523749aad5..f9dde7bc750898fdf41c1930017a5aec0000e2af 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -25,14 +25,14 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc); static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw); static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc); static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc); -static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc); -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pCreate); -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc); -static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg); -static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg); -static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew); +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate); +static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc); +static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq); +static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq); +static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq); +static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextFunc(SMnode *pMnode, void *pIter); int32_t mndInitFunc(SMnode *pMnode) { @@ -44,13 +44,13 @@ int32_t mndInitFunc(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndFuncActionUpdate, .deleteFp = (SdbDeleteFp)mndFuncActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_FUNCTION, mndProcessCreateFuncMsg); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_FUNCTION, mndProcessDropFuncMsg); - mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_FUNCTION, mndProcessRetrieveFuncMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_FUNC, mndProcessCreateFuncReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_FUNC, mndProcessDropFuncReq); + mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_FUNC, mndProcessRetrieveFuncReq); - mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndGetFuncMeta); - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndRetrieveFuncs); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNCTION, mndCancelGetNextFunc); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndGetFuncMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndRetrieveFuncs); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_FUNC, mndCancelGetNextFunc); return sdbSetTable(pMnode->pSdb, table); } @@ -73,7 +73,7 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { SDB_SET_INT8(pRaw, dataPos, pFunc->outputType, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->outputLen, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->bufSize, FUNC_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pFunc->sigature, FUNC_ENCODE_OVER) + SDB_SET_INT64(pRaw, dataPos, pFunc->signature, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->commentSize, FUNC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pFunc->codeSize, FUNC_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_ENCODE_OVER) @@ -104,13 +104,11 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { goto FUNC_DECODE_OVER; } - int32_t size = sizeof(SFuncObj) + TSDB_FUNC_COMMENT_LEN + TSDB_FUNC_CODE_LEN; - SSdbRow *pRow = sdbAllocRow(size); + SSdbRow *pRow = sdbAllocRow(sizeof(SFuncObj)); if (pRow == NULL) goto FUNC_DECODE_OVER; SFuncObj *pFunc = sdbGetRowObj(pRow); if (pFunc == NULL) goto FUNC_DECODE_OVER; - char *tmp = (char *)pFunc + sizeof(SFuncObj); int32_t dataPos = 0; SDB_GET_BINARY(pRaw, dataPos, pFunc->name, TSDB_FUNC_NAME_LEN, FUNC_DECODE_OVER) @@ -121,12 +119,18 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pFunc->outputType, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->outputLen, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->bufSize, FUNC_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pFunc->sigature, FUNC_DECODE_OVER) + SDB_GET_INT64(pRaw, dataPos, &pFunc->signature, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->commentSize, FUNC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pFunc->codeSize, FUNC_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pFunc->pData, pFunc->commentSize + pFunc->codeSize, FUNC_DECODE_OVER) - pFunc->pComment = pFunc->pData; - pFunc->pCode = (pFunc->pData + pFunc->commentSize); + + pFunc->pComment = calloc(1, pFunc->commentSize); + pFunc->pCode = calloc(1, pFunc->codeSize); + if (pFunc->pComment == NULL || pFunc->pCode == NULL) { + goto FUNC_DECODE_OVER; + } + + SDB_GET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, FUNC_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, FUNC_DECODE_OVER) terrno = 0; @@ -148,136 +152,136 @@ static int32_t mndFuncActionInsert(SSdb *pSdb, SFuncObj *pFunc) { static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { mTrace("func:%s, perform delete action, row:%p", pFunc->name, pFunc); + tfree(pFunc->pCode); + tfree(pFunc->pComment); return 0; } -static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOldFunc, SFuncObj *pNewFunc) { - mTrace("func:%s, perform update action, old row:%p new row:%p", pOldFunc->name, pOldFunc, pNewFunc); +static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { + mTrace("func:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); return 0; } -static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pMsg, SCreateFuncReq *pCreate) { - SFuncObj *pFunc = calloc(1, sizeof(SFuncObj) + pCreate->commentSize + pCreate->codeSize); - pFunc->createdTime = taosGetTimestampMs(); - pFunc->funcType = pCreate->funcType; - pFunc->scriptType = pCreate->scriptType; - pFunc->outputType = pCreate->outputType; - pFunc->outputLen = pCreate->outputLen; - pFunc->bufSize = pCreate->bufSize; - pFunc->sigature = pCreate->sigature; - pFunc->commentSize = pCreate->commentSize; - pFunc->codeSize = pCreate->codeSize; - pFunc->pComment = pFunc->pData; - memcpy(pFunc->pComment, pCreate->pCont, pCreate->commentSize); - pFunc->pCode = pFunc->pData + pCreate->commentSize; - memcpy(pFunc->pCode, pCreate->pCont + pCreate->commentSize, pFunc->codeSize); - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); - if (pTrans == NULL) { - free(pFunc); - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); - return -1; +static SFuncObj *mndAcquireFunc(SMnode *pMnode, char *funcName) { + SSdb *pSdb = pMnode->pSdb; + SFuncObj *pFunc = sdbAcquire(pSdb, SDB_FUNC, funcName); + if (pFunc == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; } + return pFunc; +} - mDebug("trans:%d, used to create func:%s", pTrans->id, pCreate->name); +static void mndReleaseFunc(SMnode *pMnode, SFuncObj *pFunc) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pFunc); +} - SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - free(pFunc); - mndTransDrop(pTrans); - return -1; +static int32_t mndCreateFunc(SMnode *pMnode, SMnodeMsg *pReq, SCreateFuncReq *pCreate) { + int32_t code = -1; + STrans *pTrans = NULL; + + SFuncObj func = {0}; + memcpy(func.name, pCreate->name, TSDB_FUNC_NAME_LEN); + func.createdTime = taosGetTimestampMs(); + func.funcType = pCreate->funcType; + func.scriptType = pCreate->scriptType; + func.outputType = pCreate->outputType; + func.outputLen = pCreate->outputLen; + func.bufSize = pCreate->bufSize; + func.signature = pCreate->signature; + func.commentSize = pCreate->commentSize; + func.codeSize = pCreate->codeSize; + func.pComment = malloc(func.commentSize); + func.pCode = malloc(func.codeSize); + if (func.pCode == NULL || func.pCode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto CREATE_FUNC_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING); - SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - free(pFunc); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED); + memcpy(func.pComment, pCreate->pCont, pCreate->commentSize); + memcpy(func.pCode, pCreate->pCont + pCreate->commentSize, func.codeSize); - SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - free(pFunc); - mndTransDrop(pTrans); - return -1; - } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); + if (pTrans == NULL) goto CREATE_FUNC_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + mDebug("trans:%d, used to create func:%s", pTrans->id, pCreate->name); + + SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto CREATE_FUNC_OVER; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto CREATE_FUNC_OVER; - free(pFunc); + SSdbRaw *pUndoRaw = mndFuncActionEncode(&func); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto CREATE_FUNC_OVER; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto CREATE_FUNC_OVER; + + SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto CREATE_FUNC_OVER; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto CREATE_FUNC_OVER; + + if (mndTransPrepare(pMnode, pTrans) != 0) goto CREATE_FUNC_OVER; + + code = 0; + +CREATE_FUNC_OVER: + free(func.pCode); + free(func.pComment); mndTransDrop(pTrans); - return 0; + return code; } -static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pMsg, SFuncObj *pFunc) { - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg); - if (pTrans == NULL) { - mError("func:%s, failed to drop since %s", pFunc->name, terrstr()); - return -1; - } +static int32_t mndDropFunc(SMnode *pMnode, SMnodeMsg *pReq, SFuncObj *pFunc) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); + if (pTrans == NULL) goto DROP_FUNC_OVER; + mDebug("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto DROP_FUNC_OVER; sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto DROP_FUNC_OVER; sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto DROP_FUNC_OVER; sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; - } + if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_FUNC_OVER; + + code = 0; +DROP_FUNC_OVER: mndTransDrop(pTrans); - return 0; + return code; } -static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessCreateFuncReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; - SCreateFuncReq *pCreate = pMsg->rpcMsg.pCont; + SCreateFuncReq *pCreate = pReq->rpcMsg.pCont; pCreate->outputLen = htonl(pCreate->outputLen); pCreate->bufSize = htonl(pCreate->bufSize); - pCreate->sigature = htobe64(pCreate->sigature); + pCreate->signature = htobe64(pCreate->signature); pCreate->commentSize = htonl(pCreate->commentSize); pCreate->codeSize = htonl(pCreate->codeSize); mDebug("func:%s, start to create", pCreate->name); - SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pCreate->name); + SFuncObj *pFunc = mndAcquireFunc(pMnode, pCreate->name); if (pFunc != NULL) { - sdbRelease(pMnode->pSdb, pFunc); - terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; - mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + mndReleaseFunc(pMnode, pFunc); + if (pCreate->igExists) { + mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); + return 0; + } else { + terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; + mError("func:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + } else if (terrno == TSDB_CODE_MND_FUNC_ALREADY_EXIST) { + mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } @@ -305,14 +309,13 @@ static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { return -1; } - if (pCreate->bufSize < 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) { + if (pCreate->bufSize <= 0 || pCreate->bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; mError("func:%s, failed to create since %s", pCreate->name, terrstr()); return -1; } - int32_t code = mndCreateFunc(pMnode, pMsg, pCreate); - + int32_t code = mndCreateFunc(pMnode, pReq, pCreate); if (code != 0) { mError("func:%s, failed to create since %s", pCreate->name, terrstr()); return -1; @@ -321,9 +324,9 @@ static int32_t mndProcessCreateFuncMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SDropFuncReq *pDrop = pMsg->rpcMsg.pCont; +static int32_t mndProcessDropFuncReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SDropFuncReq *pDrop = pReq->rpcMsg.pCont; mDebug("func:%s, start to drop", pDrop->name); @@ -333,14 +336,20 @@ static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { return -1; } - SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, pDrop->name); + SFuncObj *pFunc = mndAcquireFunc(pMnode, pDrop->name); if (pFunc == NULL) { - terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; - mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); - return -1; + if (pDrop->igNotExists) { + mDebug("func:%s, not exist, ignore not exist is set", pDrop->name); + return 0; + } else { + terrno = TSDB_CODE_MND_FUNC_NOT_EXIST; + mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); + return -1; + } } - int32_t code = mndDropFunc(pMnode, pMsg, pFunc); + int32_t code = mndDropFunc(pMnode, pReq, pFunc); + mndReleaseFunc(pMnode, pFunc); if (code != 0) { mError("func:%s, failed to drop since %s", pDrop->name, terrstr()); @@ -350,15 +359,26 @@ static int32_t mndProcessDropFuncMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndProcessRetrieveFuncReq(SMnodeMsg *pReq) { + int32_t code = -1; + SMnode *pMnode = pReq->pMnode; - SRetrieveFuncReq *pRetrieve = pMsg->rpcMsg.pCont; + SRetrieveFuncReq *pRetrieve = pReq->rpcMsg.pCont; pRetrieve->numOfFuncs = htonl(pRetrieve->numOfFuncs); + if (pRetrieve->numOfFuncs <= 0 || pRetrieve->numOfFuncs > TSDB_FUNC_MAX_RETRIEVE) { + terrno = TSDB_CODE_MND_INVALID_FUNC_RETRIEVE; + return -1; + } - int32_t size = sizeof(SRetrieveFuncRsp) + (sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN) * pRetrieve->numOfFuncs + 16384; + int32_t fsize = sizeof(SFuncInfo) + TSDB_FUNC_CODE_LEN + TSDB_FUNC_COMMENT_LEN; + int32_t size = sizeof(SRetrieveFuncRsp) + fsize * pRetrieve->numOfFuncs; SRetrieveFuncRsp *pRetrieveRsp = rpcMallocCont(size); + if (pRetrieveRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FUNC_RETRIEVE_OVER; + } + pRetrieveRsp->numOfFuncs = htonl(pRetrieve->numOfFuncs); char *pOutput = pRetrieveRsp->pFuncInfos; @@ -366,37 +386,42 @@ static int32_t mndProcessRetrieveFuncMsg(SMnodeMsg *pMsg) { char funcName[TSDB_FUNC_NAME_LEN] = {0}; memcpy(funcName, pRetrieve->pFuncNames + i * TSDB_FUNC_NAME_LEN, TSDB_FUNC_NAME_LEN); - SFuncObj *pFunc = sdbAcquire(pMnode->pSdb, SDB_FUNC, funcName); + SFuncObj *pFunc = mndAcquireFunc(pMnode, funcName); if (pFunc == NULL) { terrno = TSDB_CODE_MND_INVALID_FUNC; mError("func:%s, failed to retrieve since %s", funcName, terrstr()); - return -1; + goto FUNC_RETRIEVE_OVER; } SFuncInfo *pFuncInfo = (SFuncInfo *)pOutput; - - strncpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN); + memcpy(pFuncInfo->name, pFunc->name, TSDB_FUNC_NAME_LEN); pFuncInfo->funcType = pFunc->funcType; pFuncInfo->scriptType = pFunc->scriptType; pFuncInfo->outputType = pFunc->outputType; pFuncInfo->outputLen = htonl(pFunc->outputLen); pFuncInfo->bufSize = htonl(pFunc->bufSize); - pFuncInfo->sigature = htobe64(pFunc->sigature); + pFuncInfo->signature = htobe64(pFunc->signature); pFuncInfo->commentSize = htonl(pFunc->commentSize); pFuncInfo->codeSize = htonl(pFunc->codeSize); - memcpy(pFuncInfo->pCont, pFunc->pCode, pFunc->commentSize + pFunc->codeSize); - - pOutput += sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize; + memcpy(pFuncInfo->pCont, pFunc->pComment, pFunc->commentSize); + memcpy(pFuncInfo->pCont + pFunc->commentSize, pFunc->pCode, pFunc->codeSize); + pOutput += (sizeof(SFuncInfo) + pFunc->commentSize + pFunc->codeSize); + mndReleaseFunc(pMnode, pFunc); } - pMsg->pCont = pRetrieveRsp; - pMsg->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp); + pReq->pCont = pRetrieveRsp; + pReq->contLen = (int32_t)(pOutput - (char *)pRetrieveRsp); - return 0; + code = 0; + +FUNC_RETRIEVE_OVER: + if (code != 0) rpcFreeCont(pRetrieveRsp); + + return code; } -static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; @@ -454,7 +479,7 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *p pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, "show funcs"); + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } @@ -477,8 +502,8 @@ static void *mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t le return tDataTypes[type].name; } -static int32_t mndRetrieveFuncs(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveFuncs(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SFuncObj *pFunc = NULL; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 28fe0551c29eb98b89ec87aab757c6b557e46d73..e2ddcee0e9c8bba93854d5d61255bb34db048637 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -296,7 +296,7 @@ char *mndShowStr(int32_t showType) { return "show streamtables"; case TSDB_MGMT_TABLE_TP: return "show topics"; - case TSDB_MGMT_TABLE_FUNCTION: + case TSDB_MGMT_TABLE_FUNC: return "show functions"; default: return "undefined"; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 5ed801e3eac2279abfdbbe1da434b093308a8193..45b84286637718fa5d7782c1518f716f3e548c2b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -123,8 +123,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { goto STB_DECODE_OVER; } - int32_t size = sizeof(SStbObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); - SSdbRow *pRow = sdbAllocRow(size); + SSdbRow *pRow = sdbAllocRow(sizeof(SStbObj)); if (pRow == NULL) goto STB_DECODE_OVER; SStbObj *pStb = sdbGetRowObj(pRow); @@ -143,6 +142,9 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; pStb->pSchema = calloc(totalCols, sizeof(SSchema)); + if (pStb->pSchema == NULL) { + goto STB_DECODE_OVER; + } for (int32_t i = 0; i < totalCols; ++i) { SSchema *pSchema = &pStb->pSchema[i]; @@ -448,7 +450,7 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pReq, SMCreateStbReq *pCr stbObj.pSchema[i].colId = i + 1; } - int32_t code = 0; + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); if (pTrans == NULL) goto CREATE_STB_OVER; @@ -481,7 +483,7 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { SStbObj *pStb = mndAcquireStb(pMnode, pCreate->name); if (pStb != NULL) { - sdbRelease(pMnode->pSdb, pStb); + mndReleaseStb(pMnode, pStb); if (pCreate->igExists) { mDebug("stb:%s, already exist, ignore exist is set", pCreate->name); return 0; @@ -492,6 +494,7 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { } } else if (terrno != TSDB_CODE_MND_STB_NOT_EXIST) { mError("stb:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; } // topic should have different name with stb @@ -640,7 +643,7 @@ static int32_t mndDropStb(SMnode *pMnode, SMnodeMsg *pReq, SStbObj *pStb) { DROP_STB_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { @@ -665,7 +668,6 @@ static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { mndReleaseStb(pMnode, pStb); if (code != 0) { - terrno = code; mError("stb:%s, failed to drop since %s", pDrop->name, terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 9a573cbe2c44ca9d43fb30ef8f324a056906445f..df6a4a82f33ff1cabe6ed63916b45e935bc25b05 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -30,6 +30,8 @@ #define MND_SUBSCRIBE_VER_NUMBER 1 #define MND_SUBSCRIBE_RESERVE_SIZE 64 +static char *mndMakeSubscribeKey(char *cgroup, char *topicName); + static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); @@ -41,9 +43,10 @@ static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg); static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg); static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg); static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg); +static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg); static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, - SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic); + SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pSub); int32_t mndInitSubscribe(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_SUBSCRIBE, @@ -55,11 +58,62 @@ int32_t mndInitSubscribe(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndSubActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); - mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp); + mndSetMsgHandle(pMnode, TDMT_VND_MQ_SET_CONN_RSP, mndProcessSubscribeInternalRsp); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); + mndSetMsgHandle(pMnode, TDMT_MND_GET_SUB_EP, mndProcessGetSubEpReq); return sdbSetTable(pMnode->pSdb, table); } +static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SMqCMGetSubEpReq *pReq = (SMqCMGetSubEpReq *)pMsg->pCont; + SMqCMGetSubEpRsp rsp; + int64_t consumerId = be64toh(pReq->consumerId); + + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->pMnode, consumerId); + if (pConsumer == NULL) { + /*terrno = */ + return -1; + } + ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); + + strcpy(rsp.cgroup, pReq->cgroup); + rsp.consumerId = consumerId; + SArray *pTopics = pConsumer->topics; + int32_t sz = taosArrayGetSize(pTopics); + rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); + for (int32_t i = 0; i < sz; i++) { + SMqSubTopicEp topicEp; + SMqConsumerTopic *pConsumerTopic = taosArrayGet(pTopics, i); + strcpy(topicEp.topic, pConsumerTopic->name); + + SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, pConsumerTopic->name); + int32_t assignedSz = taosArrayGetSize(pSub->assigned); + topicEp.vgs = taosArrayInit(assignedSz, sizeof(SMqSubVgEp)); + for (int32_t j = 0; j < assignedSz; j++) { + SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i); + if (pCEp->consumerId == consumerId) { + taosArrayPush(pSub->assigned, pCEp); + } + } + if (taosArrayGetSize(topicEp.vgs) != 0) { + taosArrayPush(rsp.topics, &topicEp); + } + } + int32_t tlen = tEncodeSMqCMGetSubEpRsp(NULL, &rsp); + void *buf = malloc(tlen); + if (buf == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + void *abuf = buf; + tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); + //TODO: free rsp + pMsg->pCont = buf; + pMsg->contLen = tlen; + return 0; +} + static int32_t mndSplitSubscribeKey(char *key, char **topic, char **cgroup) { int i = 0; while (key[i] != ':') { @@ -96,25 +150,28 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { pSub->nextConsumerIdx = (pSub->nextConsumerIdx + 1) % taosArrayGetSize(pSub->availConsumer); // build msg - SMqSetCVgReq req = { - .vgId = pCEp->vgId, - .oldConsumerId = -1, - .newConsumerId = consumerId, - }; - strcpy(req.cgroup, cgroup); - strcpy(req.topicName, topic); - strcpy(req.sql, pTopic->sql); - strcpy(req.logicalPlan, pTopic->logicalPlan); - strcpy(req.physicalPlan, pTopic->physicalPlan); - memcpy(&req.msg, &pCEp->qExec, pCEp->execLen); - int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); + + SMqSetCVgReq *pReq = malloc(sizeof(SMqSetCVgReq)); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + strcpy(pReq->cgroup, cgroup); + strcpy(pReq->topicName, topic); + pReq->sql = strdup(pTopic->sql); + pReq->logicalPlan = strdup(pTopic->logicalPlan); + pReq->physicalPlan = strdup(pTopic->physicalPlan); + pReq->qmsgLen = pCEp->qmsgLen; + /*memcpy(pReq->qmsg, pCEp->qmsg, pCEp->qmsgLen);*/ + pReq->qmsg = strdup(pCEp->qmsg); + int32_t tlen = tEncodeSMqSetCVgReq(NULL, pReq); void *reqStr = malloc(tlen); if (reqStr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } void *abuf = reqStr; - tEncodeSMqSetCVgReq(abuf, &req); + tEncodeSMqSetCVgReq(&abuf, pReq); // persist msg STransAction action = {0}; @@ -128,6 +185,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { SSdbRaw *pRaw = mndSubActionEncode(pSub); mndTransAppendRedolog(pTrans, pRaw); + free(pReq); tfree(topic); tfree(cgroup); } @@ -143,30 +201,48 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { } static int mndInitUnassignedVg(SMnode *pMnode, SMqTopicObj *pTopic, SArray *unassignedVg) { - //convert phyplan to dag + // convert phyplan to dag SQueryDag *pDag = qStringToDag(pTopic->physicalPlan); - SArray *pArray; + SArray *pArray; + SArray *inner = taosArrayGet(pDag->pSubplans, 0); + SSubplan *plan = taosArrayGetP(inner, 0); + plan->execNode.inUse = 0; + strcpy(plan->execNode.epAddr[0].fqdn, "localhost"); + plan->execNode.epAddr[0].port = 6030; + plan->execNode.nodeId = 2; + plan->execNode.numOfEps = 1; + if (schedulerConvertDagToTaskList(pDag, &pArray) < 0) { return -1; } int32_t sz = taosArrayGetSize(pArray); - //convert dag to msg + // convert dag to msg for (int32_t i = 0; i < sz; i++) { SMqConsumerEp CEp; CEp.status = 0; CEp.lastConsumerHbTs = CEp.lastVgHbTs = -1; - STaskInfo* pTaskInfo = taosArrayGet(pArray, i); + STaskInfo *pTaskInfo = taosArrayGet(pArray, i); tConvertQueryAddrToEpSet(&CEp.epSet, &pTaskInfo->addr); + /*mDebug("subscribe convert ep %d %s %s %s %s %s\n", CEp.epSet.numOfEps, CEp.epSet.fqdn[0], CEp.epSet.fqdn[1], + * CEp.epSet.fqdn[2], CEp.epSet.fqdn[3], CEp.epSet.fqdn[4]);*/ CEp.vgId = pTaskInfo->addr.nodeId; + CEp.qmsg = strdup(pTaskInfo->msg->msg); + CEp.qmsgLen = strlen(CEp.qmsg) + 1; + printf("abc:\n%s\n", CEp.qmsg); + /*CEp.qmsg = malloc(CEp.qmsgLen);*/ + /*if (CEp.qmsg == NULL) {*/ + /*return -1;*/ + /*}*/ + /*memcpy(CEp.qmsg, pTaskInfo->msg->msg, pTaskInfo->msg->contentLen);*/ taosArrayPush(unassignedVg, &CEp); } - qDestroyQueryDag(pDag); + /*qDestroyQueryDag(pDag);*/ return 0; } static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsumerObj *pConsumer, - SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic) { + SMqConsumerTopic *pConsumerTopic, SMqTopicObj *pTopic, SMqConsumerEp *pCEp) { int32_t sz = taosArrayGetSize(pConsumerTopic->pVgInfo); for (int32_t i = 0; i < sz; i++) { int32_t vgId = *(int32_t *)taosArrayGet(pConsumerTopic->pVgInfo, i); @@ -178,27 +254,35 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume }; strcpy(req.cgroup, pConsumer->cgroup); strcpy(req.topicName, pTopic->name); - strcpy(req.sql, pTopic->sql); - strcpy(req.logicalPlan, pTopic->logicalPlan); - strcpy(req.physicalPlan, pTopic->physicalPlan); + req.sql = pTopic->sql; + req.logicalPlan = pTopic->logicalPlan; + req.physicalPlan = pTopic->physicalPlan; + req.qmsg = strdup(pCEp->qmsg); + req.qmsgLen = strlen(req.qmsg); int32_t tlen = tEncodeSMqSetCVgReq(NULL, &req); - void *reqStr = malloc(tlen); - if (reqStr == NULL) { + void *buf = malloc(sizeof(SMsgHead) + tlen); + if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - void *abuf = reqStr; + + SMsgHead *pMsgHead = (SMsgHead *)buf; + + pMsgHead->contLen = htonl(sizeof(SMsgHead) + tlen); + pMsgHead->vgId = htonl(vgId); + + void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncodeSMqSetCVgReq(&abuf, &req); STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgObj); - action.pCont = reqStr; - action.contLen = tlen; + action.pCont = buf; + action.contLen = sizeof(SMsgHead) + tlen; action.msgType = TDMT_VND_MQ_SET_CONN; mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { - free(reqStr); + free(buf); return -1; } } @@ -208,19 +292,18 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume void mndCleanupSubscribe(SMnode *pMnode) {} static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { + terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t tlen = tEncodeSubscribeObj(NULL, pSub); - int32_t size = tlen + MND_SUBSCRIBE_RESERVE_SIZE; + int32_t size = sizeof(int32_t) + tlen + MND_SUBSCRIBE_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); if (pRaw == NULL) goto SUB_ENCODE_OVER; void *buf = malloc(tlen); - if (buf == NULL) { - goto SUB_ENCODE_OVER; - } - void *abuf = buf; + if (buf == NULL) goto SUB_ENCODE_OVER; - tEncodeSubscribeObj(&buf, pSub); + void *abuf = buf; + tEncodeSubscribeObj(&abuf, pSub); int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, tlen, SUB_ENCODE_OVER); @@ -228,6 +311,8 @@ static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { SDB_SET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_ENCODE_OVER); SDB_SET_DATALEN(pRaw, dataPos, SUB_ENCODE_OVER); + terrno = TSDB_CODE_SUCCESS; + SUB_ENCODE_OVER: if (terrno != 0) { mError("subscribe:%s, failed to encode to raw:%p since %s", pSub->key, pRaw, terrstr()); @@ -259,9 +344,9 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; - void *buf = malloc(tlen + 1); - if (buf == NULL) goto SUB_DECODE_OVER; SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); + void *buf = malloc(tlen + 1); + if (buf == NULL) goto SUB_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); @@ -269,8 +354,10 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { goto SUB_DECODE_OVER; } + terrno = TSDB_CODE_SUCCESS; + SUB_DECODE_OVER: - if (terrno != 0) { + if (terrno != TSDB_CODE_SUCCESS) { mError("subscribe:%s, failed to decode from raw:%p since %s", pSub->key, pRaw, terrstr()); // TODO free subscribeobj tfree(pRow); @@ -379,10 +466,10 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { oldTopicName = ((SMqConsumerTopic *)taosArrayGet(oldSub, j))->name; j++; } else if (j >= oldTopicNum) { - newTopicName = taosArrayGet(newSub, i); + newTopicName = taosArrayGetP(newSub, i); i++; } else { - newTopicName = taosArrayGet(newSub, i); + newTopicName = taosArrayGetP(newSub, i); oldTopicName = ((SMqConsumerTopic *)taosArrayGet(oldSub, j))->name; int comp = compareLenPrefixedStr(newTopicName, oldTopicName); @@ -466,9 +553,11 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + char *key = mndMakeSubscribeKey(consumerGroup, newTopicName); + strcpy(pSub->key, key); // set unassigned vg mndInitUnassignedVg(pMnode, pTopic, pSub->unassignedVg); - //TODO: disable alter + // TODO: disable alter } taosArrayPush(pSub->availConsumer, &consumerId); @@ -477,16 +566,19 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { if (taosArrayGetSize(pConsumerTopic->pVgInfo) > 0) { ASSERT(taosArrayGetSize(pConsumerTopic->pVgInfo) == 1); - int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo); - // send setmsg to vnode - if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic) < 0) { - // TODO - return -1; + int32_t vgId = *(int32_t *)taosArrayGetLast(pConsumerTopic->pVgInfo); + SMqConsumerEp *pCEp = taosArrayGetLast(pSub->assigned); + if (pCEp->vgId == vgId) { + if (mndBuildMqSetConsumerVgReq(pMnode, pTrans, pConsumer, pConsumerTopic, pTopic, pCEp) < 0) { + // TODO + return -1; + } } + // send setmsg to vnode } SSdbRaw *pRaw = mndSubActionEncode(pSub); - /*sdbSetRawStatus(pRaw, SDB_STATUS_READY);*/ + sdbSetRawStatus(pRaw, SDB_STATUS_READY); mndTransAppendRedolog(pTrans, pRaw); #if 0 SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, consumerGroup, cgroupLen); @@ -519,8 +611,8 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { mndTransAppendRedolog(pTrans, pTopicRaw); #endif - mndReleaseTopic(pMnode, pTopic); - mndReleaseSubscribe(pMnode, pSub); + /*mndReleaseTopic(pMnode, pTopic);*/ + /*mndReleaseSubscribe(pMnode, pSub);*/ } } // part3. persist consumerObj diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index ac66e7d88b37a89db946e45cd642b02718c55b17..fa043cf7a0a6165c5046abdc08d650676f402470 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -60,7 +60,9 @@ void mndCleanupTopic(SMnode *pMnode) {} SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t size = sizeof(SMqTopicObj) + MND_TOPIC_RESERVE_SIZE; + int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1; + int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1; + int32_t size = sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + MND_TOPIC_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size); if (pRaw == NULL) goto TOPIC_ENCODE_OVER; @@ -74,12 +76,10 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); - int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1; - SDB_SET_INT32(pRaw, dataPos, strlen(pTopic->logicalPlan)+1, TOPIC_ENCODE_OVER); + SDB_SET_INT32(pRaw, dataPos, logicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->logicalPlan, logicalPlanLen, TOPIC_ENCODE_OVER); - int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1; - SDB_SET_INT32(pRaw, dataPos, strlen(pTopic->physicalPlan)+1, TOPIC_ENCODE_OVER); + SDB_SET_INT32(pRaw, dataPos, physicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER); @@ -135,7 +135,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto TOPIC_DECODE_OVER; } - SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len+1, TOPIC_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); pTopic->physicalPlan = calloc(len + 1, sizeof(char)); @@ -144,7 +144,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto TOPIC_DECODE_OVER; } - SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len+1, TOPIC_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER); @@ -231,6 +231,7 @@ static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *creattopReq) { } static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) { + mDebug("topic:%s to create", pCreate->name); SMqTopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); @@ -273,7 +274,7 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { return 0; } else { terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST; - mError("db:%s, failed to create since %s", createTopicReq.name, terrstr()); + mError("topic:%s, failed to create since already exists", createTopicReq.name); return -1; } } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 2301df65d78125c94253f0ce2b7e751f48e3a607..03226c74001d9f79a888a690d5509271eaa3fec4 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -390,9 +390,11 @@ static void mndTransDropActions(SArray *pArray) { } void mndTransDrop(STrans *pTrans) { - mndTransDropData(pTrans); - mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); - tfree(pTrans); + if (pTrans != NULL) { + mndTransDropData(pTrans); + mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); + tfree(pTrans); + } } static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw) { diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 3ca35d58a7cf8bbfdf29aac97390f8cf815f397f..5f6f2f3b4ff2f5eeed081a7839f3846642264cce 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -12,3 +12,4 @@ add_subdirectory(dnode) add_subdirectory(mnode) add_subdirectory(db) add_subdirectory(stb) +add_subdirectory(func) diff --git a/source/dnode/mnode/impl/test/func/CMakeLists.txt b/source/dnode/mnode/impl/test/func/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..26b0a60968d9c6f236ecfb08cbccd8c75e357411 --- /dev/null +++ b/source/dnode/mnode/impl/test/func/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. FUNC_SRC) +add_executable(mnode_test_func ${FUNC_SRC}) +target_link_libraries( + mnode_test_func + PUBLIC sut +) + +add_test( + NAME mnode_test_func + COMMAND mnode_test_func +) diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9d31ec7314061dd4d2e2bb686435a0be846c4b49 --- /dev/null +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -0,0 +1,521 @@ +/** + * @file func.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module func tests + * @version 1.0 + * @date 2022-01-24 + * + * @copyright Copyright (c) 2022 + * + */ + +#include "sut.h" + +class MndTestFunc : public ::testing::Test { + protected: + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_func", 9038); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: + void SetUp() override {} + void TearDown() override {} +}; + +Testbase MndTestFunc::test; + +TEST_F(MndTestFunc, 01_Show_Func) { + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, PATH_MAX + VARSTR_HEADER_SIZE, "comment"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_INT, 4, "aggregate"); + CHECK_SCHEMA(3, TSDB_DATA_TYPE_BINARY, TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE, "outputtype"); + CHECK_SCHEMA(4, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(5, TSDB_DATA_TYPE_INT, 4, "code_len"); + CHECK_SCHEMA(6, TSDB_DATA_TYPE_INT, 4, "bufsize"); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 0); +} + +TEST_F(MndTestFunc, 02_Create_Func) { + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, ""); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN + 1); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_COMMENT); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN - 1); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq) + 24; + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); + pReq->pCont[0] = 0; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_CODE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq) + 24; + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); + pReq->pCont[0] = 'a'; + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE); + } + + { + int32_t contLen = sizeof(SCreateFuncReq) + 24; + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->commentSize = htonl(TSDB_FUNC_COMMENT_LEN); + pReq->codeSize = htonl(TSDB_FUNC_CODE_LEN); + pReq->pCont[0] = 'a'; + pReq->bufSize = htonl(TSDB_FUNC_BUF_SIZE + 1); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_BUFSIZE); + } + + for (int32_t i = 0; i < 3; ++i) { + int32_t contLen = sizeof(SCreateFuncReq); + int32_t commentSize = TSDB_FUNC_COMMENT_LEN; + int32_t codeSize = TSDB_FUNC_CODE_LEN; + contLen = (contLen + codeSize + commentSize); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + pReq->igExists = 0; + if (i == 2) pReq->igExists = 1; + pReq->funcType = 1; + pReq->scriptType = 2; + pReq->outputType = TSDB_DATA_TYPE_SMALLINT; + pReq->outputLen = htonl(12); + pReq->bufSize = htonl(4); + pReq->signature = htobe64(5); + pReq->commentSize = htonl(commentSize); + pReq->codeSize = htonl(codeSize); + for (int32_t i = 0; i < commentSize - 1; ++i) { + pReq->pCont[i] = 'm'; + } + for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) { + pReq->pCont[i] = 'd'; + } + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (i == 0 || i == 2) { + ASSERT_EQ(pRsp->code, 0); + } else { + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_ALREADY_EXIST); + } + } + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckBinary("f1", TSDB_FUNC_NAME_LEN); + CheckBinaryByte('m', TSDB_FUNC_COMMENT_LEN); + CheckInt32(0); + CheckBinary("SMALLINT", TSDB_TYPE_STR_MAX_LEN); + CheckTimestamp(); + CheckInt32(TSDB_FUNC_CODE_LEN); + CheckInt32(4); +} + +TEST_F(MndTestFunc, 03_Retrieve_Func) { + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(1); + strcpy(pReq->pFuncNames, "f1"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; + pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + + EXPECT_STREQ(pFuncInfo->name, "f1"); + EXPECT_EQ(pFuncInfo->funcType, 1); + EXPECT_EQ(pFuncInfo->scriptType, 2); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pFuncInfo->outputLen, 12); + EXPECT_EQ(pFuncInfo->bufSize, 4); + EXPECT_EQ(pFuncInfo->signature, 5); + EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); + EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); + + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { + comments[i] = 'm'; + } + char codes[TSDB_FUNC_CODE_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { + codes[i] = 'd'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 0; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_RETRIEVE); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_RETRIEVE); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + strcpy(pReq->pFuncNames, "f2"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); + } + + { + int32_t contLen = sizeof(SCreateFuncReq); + int32_t commentSize = 1024; + int32_t codeSize = 9527; + contLen = (contLen + codeSize + commentSize); + + SCreateFuncReq* pReq = (SCreateFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f2"); + pReq->igExists = 1; + pReq->funcType = 2; + pReq->scriptType = 3; + pReq->outputType = TSDB_DATA_TYPE_BINARY; + pReq->outputLen = htonl(24); + pReq->bufSize = htonl(6); + pReq->signature = htobe64(18); + pReq->commentSize = htonl(commentSize); + pReq->codeSize = htonl(codeSize); + for (int32_t i = 0; i < commentSize - 1; ++i) { + pReq->pCont[i] = 'p'; + } + for (int32_t i = commentSize; i < commentSize + codeSize - 1; ++i) { + pReq->pCont[i] = 'q'; + } + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 1; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(1); + strcpy(pReq->pFuncNames, "f2"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; + pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + + EXPECT_STREQ(pFuncInfo->name, "f2"); + EXPECT_EQ(pFuncInfo->funcType, 2); + EXPECT_EQ(pFuncInfo->scriptType, 3); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pFuncInfo->outputLen, 24); + EXPECT_EQ(pFuncInfo->bufSize, 6); + EXPECT_EQ(pFuncInfo->signature, 18); + EXPECT_EQ(pFuncInfo->commentSize, 1024); + EXPECT_EQ(pFuncInfo->codeSize, 9527); + + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char* comments = (char*)calloc(1, 1024); + for (int32_t i = 0; i < 1024 - 1; ++i) { + comments[i] = 'p'; + } + char* codes = (char*)calloc(1, 9527); + for (int32_t i = 0; i < 9527 - 1; ++i) { + codes[i] = 'q'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); + free(comments); + free(codes); + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 2; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + strcpy(pReq->pFuncNames, "f2"); + strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f1"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp* pRetrieveRsp = (SRetrieveFuncRsp*)pRsp->pCont; + pRetrieveRsp->numOfFuncs = htonl(pRetrieveRsp->numOfFuncs); + + { + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + + EXPECT_STREQ(pFuncInfo->name, "f2"); + EXPECT_EQ(pFuncInfo->funcType, 2); + EXPECT_EQ(pFuncInfo->scriptType, 3); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_BINARY); + EXPECT_EQ(pFuncInfo->outputLen, 24); + EXPECT_EQ(pFuncInfo->bufSize, 6); + EXPECT_EQ(pFuncInfo->signature, 18); + EXPECT_EQ(pFuncInfo->commentSize, 1024); + EXPECT_EQ(pFuncInfo->codeSize, 9527); + + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char* comments = (char*)calloc(1, 1024); + for (int32_t i = 0; i < 1024 - 1; ++i) { + comments[i] = 'p'; + } + char* codes = (char*)calloc(1, 9527); + for (int32_t i = 0; i < 9527 - 1; ++i) { + codes[i] = 'q'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); + free(comments); + free(codes); + } + + { + SFuncInfo* pFuncInfo = (SFuncInfo*)(pRetrieveRsp->pFuncInfos + sizeof(SFuncInfo) + 1024 + 9527); + pFuncInfo->outputLen = htonl(pFuncInfo->outputLen); + pFuncInfo->bufSize = htonl(pFuncInfo->bufSize); + pFuncInfo->signature = htobe64(pFuncInfo->signature); + pFuncInfo->commentSize = htonl(pFuncInfo->commentSize); + pFuncInfo->codeSize = htonl(pFuncInfo->codeSize); + EXPECT_STREQ(pFuncInfo->name, "f1"); + EXPECT_EQ(pFuncInfo->funcType, 1); + EXPECT_EQ(pFuncInfo->scriptType, 2); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pFuncInfo->outputLen, 12); + EXPECT_EQ(pFuncInfo->bufSize, 4); + EXPECT_EQ(pFuncInfo->signature, 5); + EXPECT_EQ(pFuncInfo->commentSize, TSDB_FUNC_COMMENT_LEN); + EXPECT_EQ(pFuncInfo->codeSize, TSDB_FUNC_CODE_LEN); + + char* pComment = pFuncInfo->pCont; + char* pCode = pFuncInfo->pCont + pFuncInfo->commentSize; + char comments[TSDB_FUNC_COMMENT_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_COMMENT_LEN - 1; ++i) { + comments[i] = 'm'; + } + char codes[TSDB_FUNC_CODE_LEN] = {0}; + for (int32_t i = 0; i < TSDB_FUNC_CODE_LEN - 1; ++i) { + codes[i] = 'd'; + } + EXPECT_STREQ(pComment, comments); + EXPECT_STREQ(pCode, codes); + } + } + + { + int32_t contLen = sizeof(SRetrieveFuncReq); + int32_t numOfFuncs = 2; + contLen = (contLen + numOfFuncs * TSDB_FUNC_NAME_LEN); + + SRetrieveFuncReq* pReq = (SRetrieveFuncReq*)rpcMallocCont(contLen); + pReq->numOfFuncs = htonl(numOfFuncs); + strcpy(pReq->pFuncNames, "f2"); + strcpy((char*)pReq->pFuncNames + TSDB_FUNC_NAME_LEN, "f3"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC); + } +} + +TEST_F(MndTestFunc, 04_Drop_Func) { + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, ""); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_INVALID_FUNC_NAME); + } + + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f3"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_FUNC_NOT_EXIST); + } + + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f3"); + pReq->igNotExists = 1; + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + int32_t contLen = sizeof(SDropFuncReq); + + SDropFuncReq* pReq = (SDropFuncReq*)rpcMallocCont(contLen); + strcpy(pReq->name, "f1"); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + // restart + test.Restart(); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_FUNC, ""); + CHECK_META("show functions", 7); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckBinary("f2", TSDB_FUNC_NAME_LEN); +} diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 0f318dea0b409d79548dbca2ddb5721fccd4be4d..3a1e5b9c95737304d3460b0d173a8d6a6123fdf6 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -318,7 +318,7 @@ int tqRegisterContext(STqGroup*, void* ahandle); int tqSendLaunchQuery(STqMsgItem*, int64_t offset); #endif -int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp); +int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessSetConnReq(STQ* pTq, char* msg); #ifdef __cplusplus diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 362ad9b2d8f2fe66815a426147ad5ee98fc5ec86..5f5acc1b05cf4ff5c40d234b0e4cee681ad28ed1 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -75,7 +75,6 @@ typedef struct STsdbQueryCond { } STsdbQueryCond; typedef struct { - void *pTable; TSKEY lastKey; uint64_t uid; } STableKeyInfo; @@ -104,8 +103,7 @@ typedef void* tsdbReadHandleT; * @param qinfo query info handle from query processor * @return */ -tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, - void *pRef); +tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, uint64_t taskId); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. @@ -142,7 +140,7 @@ bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); */ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, - SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId); + SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId); /** * get num of rows in mem table * diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 9e6ecb6e23c5a7819651db1036cfdfdf323df636..2accfd6279a8d31c075e1c1a47f927394b77ad22 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -159,20 +159,18 @@ int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); * * @param pVnode The vnode object. * @param pMsg The request message - * @param pRsp The response message * @return int 0 for success, -1 for failure */ -int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); /** * @brief Process a fetch message. * * @param pVnode The vnode object. * @param pMsg The request message - * @param pRsp The response message * @return int 0 for success, -1 for failure */ -int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg); /* ------------------------ SVnodeCfg ------------------------ */ /** diff --git a/source/dnode/vnode/src/meta/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c index d8ee48ff23e7224821b465bea29ddcb0a508d609..dfdc144750ab568b62d82d8780982d2f9a5c32d5 100644 --- a/source/dnode/vnode/src/meta/metaBDBImpl.c +++ b/source/dnode/vnode/src/meta/metaBDBImpl.c @@ -587,7 +587,11 @@ char *metaTbCursorNext(SMTbCursor *pTbCur) { pBuf = value.data; metaDecodeTbInfo(pBuf, &tbCfg); if (tbCfg.type == META_SUPER_TABLE) { + free(tbCfg.name); + free(tbCfg.stbCfg.pTagSchema); continue; + } else if (tbCfg.type == META_CHILD_TABLE) { + kvRowFree(tbCfg.ctbCfg.pTag); } return tbCfg.name; } else { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 49bbb77797a4e01f81ee41356a77b1a3f3f39029..c164487aa2f470fffce462e8a29d24a4f4baddf4 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -81,6 +81,10 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl } void tqClose(STQ* pTq) { + if (pTq) { + tfree(pTq->path); + free(pTq); + } // TODO } @@ -663,8 +667,8 @@ int tqItemSSize() { } #endif -int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { - SMqConsumeReq* pReq = pMsg->pCont; +int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { + SMqConsumeReq* pReq = pMsg->pCont; SRpcMsg rpcMsg; int64_t reqId = pReq->reqId; int64_t consumerId = pReq->consumerId; @@ -795,9 +799,9 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { return -1; } strcpy(pTopic->topicName, req.topicName); - strcpy(pTopic->sql, req.sql); - strcpy(pTopic->logicalPlan, req.logicalPlan); - strcpy(pTopic->physicalPlan, req.physicalPlan); + pTopic->sql = strdup(req.sql); + pTopic->logicalPlan = strdup(req.logicalPlan); + pTopic->physicalPlan = strdup(req.physicalPlan); pTopic->buffer.firstOffset = -1; pTopic->buffer.lastOffset = -1; @@ -807,9 +811,10 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); - pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(&req.msg, pReadHandle); + pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(req.qmsg, pReadHandle); } taosArrayPush(pConsumer->topics, pTopic); + terrno = TSDB_CODE_SUCCESS; return 0; } @@ -822,7 +827,7 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { pReadHandle->pMsg = NULL; pReadHandle->ver = -1; pReadHandle->pColIdList = NULL; - return NULL; + return pReadHandle; } void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ver) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index ca2b1069f9fac80faa90785df4a3050e65d624b5..be46f63168c7416a80058c26b9102c97617ea5db 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -359,7 +359,7 @@ void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd) { tsdbResetFSStatus(pfs->nstatus); pfs->nstatus->meta = pfs->cstatus->meta; // if (pfs->cstatus->pmf == NULL) { - pfs->nstatus->meta.version = 0; + pfs->nstatus->meta.version += 1; // } else { // pfs->nstatus->meta.version = pfs->cstatus->meta.version + 1; // } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 68c7f190db6f53529d70f94611501cb3db25f9f6..1bca7aea1028f115c1f4752e5387c16e719cf417 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -81,7 +81,6 @@ enum { CHECKINFO_CHOSEN_BOTH = 2 //for update=2(merge case) }; - typedef struct STableCheckInfo { uint64_t tableId; TSKEY lastKey; @@ -132,7 +131,7 @@ typedef struct STsdbReadHandle { bool loadExternalRow; // load time window external data rows bool currentLoadExternalRows; // current load external rows int32_t loadType; // block load type - uint64_t qId; // query info handle, for debug purpose + char *idStr; // query info handle, for debug purpose int32_t type; // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows SDFileSet* pFileGroup; SFSIter fileIter; @@ -209,31 +208,6 @@ static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool load return pLocalIdList; } -static void tsdbMayTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle, SArray* psTable) { -// assert(pTsdbReadHandle != NULL && pTsdbReadHandle->pMemRef != NULL); -// -// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; -// if (pTsdbReadHandle->pMemRef->ref++ == 0) { -// tsdbTakeMemSnapshot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot), psTable); -// } -// -// taosArrayDestroy(psTable); -} - -static void tsdbMayUnTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle) { -// assert(pTsdbReadHandle != NULL); -// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; -// if (pMemRef == NULL) { // it has been freed -// return; -// } -// -// if (--pMemRef->ref == 0) { -// tsdbUnTakeMemSnapShot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot)); -// } -// -// pTsdbReadHandle->pMemRef = NULL; -} - //int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle) { // STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; // @@ -263,9 +237,9 @@ static void tsdbMayUnTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle) { // return rows; //} -static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList, SArray** psTable) { - size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList); - assert(sizeOfGroup >= 1); +static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList) { + size_t numOfGroup = taosArrayGetSize(pGroupList->pGroupList); + assert(numOfGroup >= 1); // allocate buffer in order to load data blocks from file SArray* pTableCheckInfo = taosArrayInit(pGroupList->numOfTables, sizeof(STableCheckInfo)); @@ -273,14 +247,8 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S return NULL; } - SArray* pTable = taosArrayInit(4, sizeof(STable*)); - if (pTable == NULL) { - taosArrayDestroy(pTableCheckInfo); - return NULL; - } - // todo apply the lastkey of table check to avoid to load header file - for (int32_t i = 0; i < sizeOfGroup; ++i) { + for (int32_t i = 0; i < numOfGroup; ++i) { SArray* group = *(SArray**) taosArrayGet(pGroupList->pGroupList, i); size_t gsize = taosArrayGetSize(group); @@ -289,12 +257,7 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S for (int32_t j = 0; j < gsize; ++j) { STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); - STableCheckInfo info = { .lastKey = pKeyInfo->lastKey}; -// assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || -// info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); - - info.tableId = pKeyInfo->uid; - + STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .tableId = pKeyInfo->uid}; if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReadHandle->window.skey) { info.lastKey = pTsdbReadHandle->window.skey; @@ -306,17 +269,12 @@ static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, S } taosArrayPush(pTableCheckInfo, &info); - tsdbDebug("%p check table uid:%"PRId64" from lastKey:%"PRId64" 0x%"PRIx64, pTsdbReadHandle, info.tableId, info.lastKey, pTsdbReadHandle->qId); + tsdbDebug("%p check table uid:%"PRId64" from lastKey:%"PRId64" %s", pTsdbReadHandle, info.tableId, info.lastKey, pTsdbReadHandle->idStr); } } -// taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); - size_t gsize = taosArrayGetSize(pTableCheckInfo); -// for (int32_t i = 0; i < gsize; ++i) { -// STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i); -// } - - *psTable = pTable; + // TODO group table according to the tag value. + taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); return pTableCheckInfo; } @@ -389,13 +347,13 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* } if (updateTs) { - tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64 - ", 0x%" PRIx64, pTsdbReadHandle, pCond->twindow.skey, pCond->twindow.ekey, pTsdbReadHandle->window.skey, - pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); + tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64 ", %s", + pTsdbReadHandle, pCond->twindow.skey, pCond->twindow.ekey, pTsdbReadHandle->window.skey, + pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); } } -static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, STsdbMemTable* pMemRef) { +static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, uint64_t taskId) { STsdbReadHandle* pReadHandle = calloc(1, sizeof(STsdbReadHandle)); if (pReadHandle == NULL) { goto _end; @@ -408,7 +366,6 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, pReadHandle->cur.win = TSWINDOW_INITIALIZER; pReadHandle->checkFiles = true; pReadHandle->activeIndex = 0; // current active table index - pReadHandle->qId = qId; pReadHandle->allocSize = 0; pReadHandle->locateStart = false; pReadHandle->loadType = pCond->type; @@ -417,6 +374,10 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, pReadHandle->loadExternalRow = pCond->loadExternalRows; pReadHandle->currentLoadExternalRows = pCond->loadExternalRows; + char buf[128] = {0}; + snprintf(buf, tListLen(buf), "TID:0x%"PRIx64" QID:0x%"PRIx64, taskId, qId); + pReadHandle->idStr = strdup(buf); + if (tsdbInitReadH(&pReadHandle->rhelper, (STsdb*)tsdb) != 0) { goto _end; } @@ -455,7 +416,7 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->config.maxRowsPerFileBlock); if (pReadHandle->pDataCols == NULL) { - tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pReadHandle, pReadHandle->qId); + tsdbError("%p failed to malloc buf for pDataCols, %s", pReadHandle, pReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _end; } @@ -471,8 +432,8 @@ static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, return NULL; } -tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, void* pRef) { - STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); +tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId) { + STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; } @@ -481,20 +442,17 @@ tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroup return (tsdbReadHandleT*) pTsdbReadHandle; } - SArray* psTable = NULL; - // todo apply the lastkey of table check to avoid to load header file - pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, &psTable); + pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList); if (pTsdbReadHandle->pTableCheckInfo == NULL) { // tsdbCleanupQueryHandle(pTsdbReadHandle); - taosArrayDestroy(psTable); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } -// tsdbMayTakeMemSnapshot(pTsdbReadHandle, psTable); + tsdbDebug("%p total numOfTable:%" PRIzu " in this query, group %"PRIzu" %s", pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), + taosArrayGetSize(groupList->pGroupList), pTsdbReadHandle->idStr); - tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), pTsdbReadHandle->qId); return (tsdbReadHandleT) pTsdbReadHandle; } @@ -575,7 +533,7 @@ void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond // pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); } -tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { +tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table @@ -583,7 +541,7 @@ tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroup return NULL; } - STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, taskId); if (pTsdbReadHandle == NULL) { return NULL; } @@ -666,7 +624,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr return pNew; } -tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pRef) { +tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, uint64_t taskId) { STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); if (pNew->numOfTables == 0) { @@ -681,7 +639,7 @@ tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond } } - STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, taskId); pTsdbReadHandle->loadExternalRow = true; pTsdbReadHandle->currentLoadExternalRows = true; @@ -734,8 +692,8 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); TSKEY key = memRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, - pHandle, pCheckInfo->tableId, key, order, (*pMem)->keyMin, (*pMem)->keyMax, pCheckInfo->lastKey, (*pMem)->nrows, pHandle->qId); + "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %s", + pHandle, pCheckInfo->tableId, key, order, (*pMem)->keyMin, (*pMem)->keyMax, pCheckInfo->lastKey, (*pMem)->nrows, pHandle->idStr); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -744,7 +702,7 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe } } else { - tsdbDebug("%p uid:%"PRId64", no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId); + tsdbDebug("%p uid:%"PRId64", no data in mem, %s", pHandle, pCheckInfo->tableId, pHandle->idStr); } if (!imemEmpty) { @@ -754,8 +712,8 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); TSKEY key = memRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, - pHandle, pCheckInfo->tableId, key, order, (*pIMem)->keyMin, (*pIMem)->keyMax, pCheckInfo->lastKey, (*pIMem)->nrows, pHandle->qId); + "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %s", + pHandle, pCheckInfo->tableId, key, order, (*pIMem)->keyMin, (*pIMem)->keyMax, pCheckInfo->lastKey, (*pIMem)->nrows, pHandle->idStr); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -763,7 +721,7 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe assert(pCheckInfo->lastKey >= key); } } else { - tsdbDebug("%p uid:%"PRId64", no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId); + tsdbDebug("%p uid:%"PRId64", no data in imem, %s", pHandle, pCheckInfo->tableId, pHandle->idStr); } return true; @@ -952,8 +910,8 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { } pCheckInfo->lastKey = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64", check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, - pCheckInfo->tableId, pCheckInfo->lastKey, pHandle->order, pHandle->qId); + tsdbDebug("%p uid:%" PRId64", check data in buffer from skey:%" PRId64 ", order:%d, %s", pHandle, + pCheckInfo->tableId, pCheckInfo->lastKey, pHandle->order, pHandle->idStr); // all data in mem are checked already. if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) || @@ -1138,21 +1096,21 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl STSchema *pSchema = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, pCheckInfo->tableId, 0); int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); + tsdbError("%p failed to malloc buf for pDataCols, %s", pTsdbReadHandle, pTsdbReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[0], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], %s", pTsdbReadHandle, pTsdbReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[1], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], %s", pTsdbReadHandle, pTsdbReadHandle->idStr); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } @@ -1188,15 +1146,15 @@ static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBl int64_t elapsedTime = (taosGetTimestampUs() - st); pTsdbReadHandle->cost.blockLoadTime += elapsedTime; - tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, 0x%"PRIx64, - pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pTsdbReadHandle->qId); + tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, %s", + pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pTsdbReadHandle->idStr); return TSDB_CODE_SUCCESS; _error: pBlock->numOfRows = 0; - tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, 0x%"PRIx64, - pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->qId); + tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, %s", + pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->idStr); return terrno; } @@ -1219,9 +1177,9 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update); if (key != TSKEY_INITIAL_VAL) { - tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pTsdbReadHandle, key, pTsdbReadHandle->qId); + tsdbDebug("%p key in mem:%"PRId64", %s", pTsdbReadHandle, key, pTsdbReadHandle->idStr); } else { - tsdbDebug("%p no data in mem, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); + tsdbDebug("%p no data in mem, %s", pTsdbReadHandle, pTsdbReadHandle->idStr); } if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || @@ -1288,11 +1246,11 @@ static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* assert(cur->blockCompleted); if (cur->rows == binfo.rows) { - tsdbDebug("%p whole file block qualified, brange:%"PRId64"-%"PRId64", rows:%d, lastKey:%"PRId64", %"PRIx64, - pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pTsdbReadHandle->qId); + tsdbDebug("%p whole file block qualified, brange:%"PRId64"-%"PRId64", rows:%d, lastKey:%"PRId64", %s", + pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pTsdbReadHandle->idStr); } else { - tsdbDebug("%p create data block from remain file block, brange:%"PRId64"-%"PRId64", rows:%d, total:%d, lastKey:%"PRId64", %"PRIx64, - pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pTsdbReadHandle->qId); + tsdbDebug("%p create data block from remain file block, brange:%"PRId64"-%"PRId64", rows:%d, total:%d, lastKey:%"PRId64", %s", + pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pTsdbReadHandle->idStr); } } @@ -1815,8 +1773,8 @@ static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STa updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pTsdbReadHandle); - tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, - pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId); + tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %s", + pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->idStr); } int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo) { @@ -1868,9 +1826,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo); tsdbDebug("%p uid:%" PRIu64" start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," - "end:%d, 0x%"PRIx64, + "end:%d, %s", pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, - blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->qId); + blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->idStr); // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; @@ -2027,8 +1985,8 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pTsdbReadHandle); - tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, - pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId); + tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %s", + pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->idStr); } int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { @@ -2202,13 +2160,13 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu memcpy(pTsdbReadHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); cleanBlockOrderSupporter(&sup, numOfQualTables); - tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pTsdbReadHandle, cnt, - pTsdbReadHandle->qId); + tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted %s", pTsdbReadHandle, cnt, + pTsdbReadHandle->idStr); return TSDB_CODE_SUCCESS; } - tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pTsdbReadHandle, cnt, - numOfQualTables, pTsdbReadHandle->qId); + tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pTsdbReadHandle, cnt, + numOfQualTables, pTsdbReadHandle->idStr); assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 sup.numOfTables = numOfQualTables; @@ -2244,7 +2202,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu * } */ - tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pTsdbReadHandle, cnt, pTsdbReadHandle->qId); + tsdbDebug("%p %d data blocks sort completed, %s", pTsdbReadHandle, cnt, pTsdbReadHandle->idStr); cleanBlockOrderSupporter(&sup, numOfTables); free(pTree); @@ -2302,8 +2260,8 @@ static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exi if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle, - pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pTsdbReadHandle, + pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); pTsdbReadHandle->pFileGroup = NULL; assert(pTsdbReadHandle->numOfBlocks == 0); break; @@ -2326,8 +2284,8 @@ static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exi break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables, - pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pTsdbReadHandle, numOfBlocks, numOfTables, + pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->idStr); assert(numOfBlocks >= 0); if (numOfBlocks == 0) { @@ -2420,8 +2378,8 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist* if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle, - pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pTsdbReadHandle, + pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); pTsdbReadHandle->pFileGroup = NULL; break; } @@ -2444,8 +2402,8 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist* break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables, - pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pTsdbReadHandle, numOfBlocks, numOfTables, + pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->idStr); if (numOfBlocks == 0) { continue; @@ -2499,8 +2457,8 @@ static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exis if ((!cur->mixBlock) || cur->blockCompleted) { // all data blocks in current file has been checked already, try next file if exists } else { - tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pTsdbReadHandle, cur->slot, cur->pos, - pTsdbReadHandle->qId); + tsdbDebug("%p continue in current data block, index:%d, pos:%d, %s", pTsdbReadHandle, cur->slot, cur->pos, + pTsdbReadHandle->idStr); int32_t code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlockInfo->compBlock, pCheckInfo); *exists = (pTsdbReadHandle->realNumOfRows > 0); @@ -2624,8 +2582,8 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int } int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pTsdbReadHandle, - elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->qId); + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %s", pTsdbReadHandle, + elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->idStr); return numOfRows; } @@ -2639,7 +2597,7 @@ static int32_t getAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { break; } - STableKeyInfo info = {.pTable = NULL, .lastKey = TSKEY_INITIAL_VAL, uid = id}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id}; taosArrayPush(list, &info); } @@ -2937,7 +2895,7 @@ bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; if (emptyQueryTimewindow(pTsdbReadHandle)) { - tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); + tsdbDebug("%p query window not overlaps with the data set, no result returned, %s", pTsdbReadHandle, pTsdbReadHandle->idStr); return false; } @@ -3052,7 +3010,7 @@ bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { // memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); // } // -// pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->qId, pMemRef); +// pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->idStr, pMemRef); // tfree(cond.colList); // // // current table, only one table @@ -3230,7 +3188,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { if (key < lastKey) { key = lastKey; - keyInfo.pTable = pInfo->pTable; +// keyInfo.pTable = pInfo->pTable; keyInfo.lastKey = key; pInfo->lastKey = key; @@ -3244,29 +3202,19 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { } } - // clear current group, unref unused table - for (int32_t i = 0; i < numOfTables; ++i) { - STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(pGroup, i); - - // keyInfo.pTable may be NULL here. - if (pInfo->pTable != keyInfo.pTable) { -// tsdbUnRefTable(pInfo->pTable); - } - } - // more than one table in each group, only one table left for each group - if (keyInfo.pTable != NULL) { - totalNumOfTable++; - if (taosArrayGetSize(pGroup) == 1) { - // do nothing - } else { - taosArrayClear(pGroup); - taosArrayPush(pGroup, &keyInfo); - } - } else { // mark all the empty groups, and remove it later - taosArrayDestroy(pGroup); - taosArrayPush(emptyGroup, &j); - } +// if (keyInfo.pTable != NULL) { +// totalNumOfTable++; +// if (taosArrayGetSize(pGroup) == 1) { +// // do nothing +// } else { +// taosArrayClear(pGroup); +// taosArrayPush(pGroup, &keyInfo); +// } +// } else { // mark all the empty groups, and remove it later +// taosArrayDestroy(pGroup); +// taosArrayPush(emptyGroup, &j); +// } } // window does not being updated, so set the original @@ -3456,11 +3404,13 @@ void filterPrepare(void* expr, void* param) { } } +#endif static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { +#if 0 STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; - STable* pTable1 = ((STableKeyInfo*) p1)->pTable; - STable* pTable2 = ((STableKeyInfo*) p2)->pTable; + STable* pTable1 = ((STableKeyInfo*) p1)->uid; + STable* pTable2 = ((STableKeyInfo*) p2)->uid; for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) { SColIndex* pColIndex = &pTableGroupSupp->pCols[i]; @@ -3508,10 +3458,9 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa return ret; } } - +#endif return 0; } -#endif static int tsdbCheckInfoCompar(const void* key1, const void* key2) { if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) { @@ -3527,10 +3476,9 @@ static int tsdbCheckInfoCompar(const void* key1, const void* key2) { void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) { STable* pTable = taosArrayGetP(pTableList, 0); - SArray* g = taosArrayInit(16, sizeof(STableKeyInfo)); - STableKeyInfo info = {.pTable = pTable, .lastKey = skey}; + STableKeyInfo info = {.lastKey = skey}; taosArrayPush(g, &info); for (int32_t i = 1; i < numOfTables; ++i) { @@ -3541,13 +3489,13 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable assert(ret == 0 || ret == -1); if (ret == 0) { - STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; + STableKeyInfo info1 = {.lastKey = skey}; taosArrayPush(g, &info1); } else { taosArrayPush(pGroups, &g); // current group is ended, start a new group g = taosArrayInit(16, sizeof(STableKeyInfo)); - STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; + STableKeyInfo info1 = {.lastKey = skey}; taosArrayPush(g, &info1); } } @@ -3580,8 +3528,8 @@ SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColInd sup.pTagSchema = pTagSchema->pSchema; sup.pCols = pCols; -// taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); -// createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn); + taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); + createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn); } return pTableGroup; @@ -3689,16 +3637,16 @@ SArray* createTableGroup(SArray* pTableList, SSchemaWrapper* pTagSchema, SColInd int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, - SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId) { + SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) { STbCfg* pTbCfg = metaGetTbInfoByUid(tsdb->pMeta, uid); if (pTbCfg == NULL) { - tsdbError("%p failed to get stable, uid:%"PRIu64", reqId:0x%"PRIx64, tsdb, uid, reqId); + tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; goto _error; } if (pTbCfg->type != META_SUPER_TABLE) { - tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", reId:0x%"PRIx64, tsdb, uid, reqId); + tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); terrno = TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client goto _error; } @@ -3717,8 +3665,8 @@ int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const ch pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu", tsdb, - pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); + tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, + pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId); taosArrayDestroy(res); return ret; @@ -3891,7 +3839,7 @@ void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { tfree(pTsdbReadHandle->statis); if (!emptyQueryTimewindow(pTsdbReadHandle)) { - tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); +// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); } else { assert(pTsdbReadHandle->pTableCheckInfo == NULL); } @@ -3910,8 +3858,8 @@ void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { SIOCostSummary* pCost = &pTsdbReadHandle->cost; - tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64, - pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->qId); + tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %s", + pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->idStr); tfree(pTsdbReadHandle); } diff --git a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c index 5844ef03452156217bb3b8ab6d2ac5975a944929..14b9a5124ffcb89f91194190b661f5e70062c98f 100644 --- a/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c +++ b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c @@ -114,4 +114,4 @@ static void vArenaNodeFree(SVArenaNode *pNode) { if (pNode) { free(pNode); } -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/vnd/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c index 8df6b4256652c0b9c471b43ef299f4cca7279a8a..434498eef59bdbaa1fa95ec7d6c63f262c2057bc 100644 --- a/source/dnode/vnode/src/vnd/vnodeBufferPool.c +++ b/source/dnode/vnode/src/vnd/vnodeBufferPool.c @@ -71,6 +71,7 @@ int vnodeOpenBufPool(SVnode *pVnode) { void vnodeCloseBufPool(SVnode *pVnode) { if (pVnode->pBufPool) { + tfree(pVnode->pBufPool->pMAF); vmaDestroy(pVnode->pBufPool->inuse); while (true) { diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index 730155b75a3eeb9d37341b5fca25b8b6b2008ae7..d7628441206ac9bd9a6945d9813811e402ed5b5e 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -98,6 +98,8 @@ int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { /* ------------------------ STATIC METHODS ------------------------ */ static void* loop(void* arg) { + setThreadName("vnode-commit"); + SVnodeTask* pTask; for (;;) { pthread_mutex_lock(&(vnodeMgr.mutex)); @@ -119,6 +121,7 @@ static void* loop(void* arg) { pthread_mutex_unlock(&(vnodeMgr.mutex)); (*(pTask->execute))(pTask->arg); + free(pTask); } return NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 888520dc56d847680c7283105c8278cbe8481719..b2118942815e01632262bbfa03671034b7c51683 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -17,14 +17,14 @@ #include "vnd.h" static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); -static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg); int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode, (putReqToQueryQFp)vnodePutReqToVQueryQ); } -int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in query queue is processing"); switch (pMsg->msgType) { @@ -38,7 +38,7 @@ int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } } -int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in fetch queue is processing"); switch (pMsg->msgType) { case TDMT_VND_FETCH: @@ -57,24 +57,24 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { return vnodeGetTableList(pVnode, pMsg); // return qWorkerProcessShowFetchMsg(pVnode->pMeta, pVnode->pQuery, pMsg); case TDMT_VND_TABLE_META: - return vnodeGetTableMeta(pVnode, pMsg, pRsp); + return vnodeGetTableMeta(pVnode, pMsg); case TDMT_VND_CONSUME: - return tqProcessConsumeReq(pVnode->pTq, pMsg, pRsp); + return tqProcessConsumeReq(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; } } -static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { +static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { STableInfoReq * pReq = (STableInfoReq *)(pMsg->pCont); STbCfg * pTbCfg = NULL; STbCfg * pStbCfg = NULL; tb_uid_t uid; int32_t nCols; int32_t nTagCols; - SSchemaWrapper *pSW; - STableMetaRsp * pTbMetaMsg = NULL; + SSchemaWrapper *pSW = NULL; + STableMetaRsp *pTbMetaMsg = NULL; SSchema * pTagSchema; SRpcMsg rpcMsg; int msgLen = 0; @@ -145,6 +145,22 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { _exit: + if (pSW != NULL) { + tfree(pSW->pSchema); + tfree(pSW); + } + + if (pTbCfg) { + tfree(pTbCfg->name); + if (pTbCfg->type == META_SUPER_TABLE) { + free(pTbCfg->stbCfg.pTagSchema); + } else if (pTbCfg->type == META_SUPER_TABLE) { + kvRowFree(pTbCfg->ctbCfg.pTag); + } + + tfree(pTbCfg); + } + rpcMsg.handle = pMsg->handle; rpcMsg.ahandle = pMsg->ahandle; rpcMsg.pCont = pTbMetaMsg; @@ -156,8 +172,8 @@ _exit: return 0; } -static void freeItemHelper(void* pItem) { - char* p = *(char**)pItem; +static void freeItemHelper(void *pItem) { + char *p = *(char **)pItem; free(p); } @@ -187,14 +203,14 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { // TODO: temp debug, and should del when show tables command ok vInfo("====vgId:%d, numOfTables: %d", pVnode->vgId, numOfTables); if (numOfTables > 10000) { - numOfTables = 10000; + numOfTables = 10000; } metaCloseTbCursor(pCur); int32_t rowLen = (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 2 + (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) + 8 + 4; - //int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); + // int32_t numOfTables = (int32_t)taosArrayGetSize(pArray); int32_t payloadLen = rowLen * numOfTables; // SVShowTablesFetchReq *pFetchReq = pMsg->pCont; @@ -208,6 +224,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { STR_TO_VARSTR(p, n); p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE); + // free(n); } pFetchRsp->numOfRows = htonl(numOfTables); @@ -222,6 +239,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) { }; rpcSendResponse(&rpcMsg); + taosArrayDestroyEx(pArray, freeItemHelper); return 0; } diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index ccddfd56d8f89832790b364643981a62dba4cee7..326d99ddbbc6e912596324ac44c3ac5aa4f0eb75 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -34,7 +34,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void * pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; taosEncodeFixedU64(&pBuf, ver); @@ -53,7 +53,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq vCreateTbReq; SVCreateTbBatchReq vCreateTbBatchReq; - void *ptr = vnodeMalloc(pVnode, pMsg->contLen); + void * ptr = vnodeMalloc(pVnode, pMsg->contLen); if (ptr == NULL) { // TODO: handle error } @@ -76,6 +76,9 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } // TODO: maybe need to clear the requst struct + free(vCreateTbReq.stbCfg.pSchema); + free(vCreateTbReq.stbCfg.pTagSchema); + free(vCreateTbReq.name); break; case TDMT_VND_CREATE_TABLE: tSVCreateTbBatchReqDeserialize(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &vCreateTbBatchReq); @@ -112,7 +115,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } break; case TDMT_VND_MQ_SET_CONN: { - if (tqProcessSetConnReq(pVnode->pTq, ptr) < 0) { + if (tqProcessSetConnReq(pVnode->pTq, POINTER_SHIFT(ptr, sizeof(SMsgHead))) < 0) { + // TODO: handle error } } break; default: @@ -129,7 +133,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // TODO: handle error } } - + return 0; } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f552afcfbf245773931fca13bd2d7621c62c1887..130e46fc4cb2a7abe2fcdfad2004dbdf1a3b47e6 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -38,7 +38,7 @@ #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) #define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES) -#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.queryId) +#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) #define curTimeWindowIndex(_winres) ((_winres)->curIndex) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index ebf3d83a1a39380828624d896109656e53ef4aba..7af2e9f14f6fb7d68723cc073fda33783d172539 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -239,7 +239,7 @@ typedef struct STaskIdInfo { uint64_t queryId; // this is also a request id uint64_t subplanId; uint64_t templateId; - uint64_t taskId; // this is a subplan id + char *str; } STaskIdInfo; typedef struct SExecTaskInfo { @@ -250,9 +250,8 @@ typedef struct SExecTaskInfo { STaskCostInfo cost; int64_t owner; // if it is in execution int32_t code; - + uint64_t totalRows; // total number of rows STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure - pthread_mutex_t lock; // used to synchronize the rsp/query threads char *sql; // query sql string jmp_buf env; // struct SOperatorInfo *pRoot; @@ -378,8 +377,10 @@ typedef struct SExchangeInfo { SSDataBlock *pResult; int32_t current; uint64_t rowsOfCurrentSource; - uint64_t bytes; // total load bytes from remote - uint64_t totalRows; + + uint64_t totalSize; // total load bytes from remote + uint64_t totalRows; // total number of rows + uint64_t totalElapsed;// total elapsed time } SExchangeInfo; typedef struct STableScanInfo { @@ -622,8 +623,6 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableReq *pQueryMsg, int32_t nu int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters); SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableReq *pQueryMsg, SColIndex *pColIndex, int32_t *code); -SQInfo *createQInfoImpl(SQueryTableReq *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo); int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start, int32_t prevResultLen, void* merger); @@ -639,32 +638,29 @@ int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); bool isTaskKilled(SExecTaskInfo *pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); bool checkNeedToCompressQueryCol(SQInfo *pQInfo); -bool doBuildResCheck(SQInfo* pQInfo); void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status); bool onlyQueryTags(STaskAttr* pQueryAttr); //void destroyUdfInfo(struct SUdfInfo* pUdfInfo); -bool isValidQInfo(void *param); - int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen); size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows); -void setQueryKilled(SQInfo *pQInfo); +void setTaskKilled(SExecTaskInfo *pTaskInfo); void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType); void publishQueryAbortEvent(SExecTaskInfo * pTaskInfo, int32_t code); void calculateOperatorProfResults(SQInfo* pQInfo); -void queryCostStatis(SQInfo *pQInfo); +void queryCostStatis(SExecTaskInfo *pTaskInfo); -void doDestroyTask(SQInfo *pQInfo); +void doDestroyTask(SExecTaskInfo *pTaskInfo); void freeQueryAttr(STaskAttr *pQuery); int32_t getMaximumIdleDurationSec(); void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status); -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle); +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle, uint64_t taskId); #endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index f9e61f91de8a30b60ba8131e8a29ba3dfd9e7aca..97a055774851d5ac6a95de581188d1866bb1af0b 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -121,11 +121,19 @@ static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputDat } static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) { - if (taosQueueSize(pDispatcher->pDataBlocks) >= pDispatcher->pManager->cfg.maxDataBlockNumPerQuery) { + uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery; + if (taosQueueSize(pDispatcher->pDataBlocks) > capacity) { + qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity, + taosQueueSize(pDispatcher->pDataBlocks)); return false; } + pBuf->allocSize = DATA_META_LENGTH(pInput->pTableRetrieveTsMap) + pDispatcher->schema.resultRowSize * pInput->pData->info.rows; pBuf->pData = malloc(pBuf->allocSize); + if (pBuf->pData == NULL) { + qError("SinkNode failed to malloc memory, size:%d, code:%d", pBuf->allocSize, TAOS_SYSTEM_ERROR(errno)); + } + return NULL != pBuf->pData; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 2f1f40813c2b4f400e1e5a33b2ff3b9421f49d5b..a32268bf4723b928c0f52a5915045283c0a4f6c2 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -18,20 +18,20 @@ #include "executorimpl.h" #include "planner.h" -static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, uint64_t reqId) { +static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) { ASSERT(pOperator != NULL); if (pOperator->operatorType != OP_StreamScan) { if (pOperator->numOfDownstream == 0) { - qError("failed to find stream scan operator to set the input data block, reqId:0x%" PRIx64, reqId); + qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } if (pOperator->numOfDownstream > 1) { // not handle this in join query - qError("join not supported for stream block scan, reqId:0x%" PRIx64, reqId); + qError("join not supported for stream block scan, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } - return doSetStreamBlock(pOperator->pDownstream[0], input, reqId); + return doSetStreamBlock(pOperator->pDownstream[0], input, id); } else { SStreamBlockScanInfo* pInfo = pOperator->info; tqReadHandleSetMsg(pInfo->readerHandle, input, 0); @@ -52,16 +52,16 @@ int32_t qSetStreamInput(qTaskInfo_t tinfo, void* input) { int32_t code = doSetStreamBlock(pTaskInfo->pRoot, input, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { - qError("failed to set the stream block data, reqId:0x%"PRIx64, GET_TASKID(pTaskInfo)); + qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); } else { - qDebug("set the stream block successfully, reqId:0x%"PRIx64, GET_TASKID(pTaskInfo)); + qDebug("%s set the stream block successfully", GET_TASKID(pTaskInfo)); } return code; } -qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* streamReadHandle) { - if (pMsg == NULL || streamReadHandle == NULL) { +qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { + if (msg == NULL || streamReadHandle == NULL) { return NULL; } @@ -74,14 +74,14 @@ qTaskInfo_t qCreateStreamExecTaskInfo(SSubQueryMsg* pMsg, void* streamReadHandle #endif struct SSubplan* plan = NULL; - int32_t code = qStringToSubplan(pMsg->msg, &plan); + int32_t code = qStringToSubplan(msg, &plan); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(streamReadHandle, 0, plan, &pTaskInfo, NULL); + code = qCreateExecTask(streamReadHandle, 0, 0, plan, &pTaskInfo, NULL); if (code != TSDB_CODE_SUCCESS) { // TODO: destroy SSubplan & pTaskInfo terrno = code; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index e5d56aca154c7730e75764c3f80a7244a69aabcb..8635b2d5c812c479ccd08b963c1e24daa9ab08a2 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -69,11 +69,11 @@ void freeParam(STaskParam *param) { tfree(param->prevResult); } -int32_t qCreateExecTask(void* readHandle, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { +int32_t qCreateExecTask(void* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { assert(readHandle != NULL && pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle); + int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -83,8 +83,9 @@ int32_t qCreateExecTask(void* readHandle, int32_t vgId, SSubplan* pSubplan, qTas if (code != TSDB_CODE_SUCCESS) { goto _error; } - - code = dsCreateDataSinker(pSubplan->pDataSink, handle); + if (handle) { + code = dsCreateDataSinker(pSubplan->pDataSink, handle); + } _error: // if failed to add ref for all tables in this query, abort current query @@ -140,16 +141,10 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); - // todo: remove it. - if (tinfo == NULL) { - return TSDB_CODE_SUCCESS; - } - *pRes = NULL; - int64_t curOwner = 0; if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { - qError("QInfo:0x%" PRIx64 "-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, + qError("%s-%p execTask is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*)curOwner); pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; return pTaskInfo->code; @@ -160,7 +155,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { } if (isTaskKilled(pTaskInfo)) { - qDebug("QInfo:0x%" PRIx64 " it is already killed, abort", GET_TASKID(pTaskInfo)); + qDebug("%s already killed, abort", GET_TASKID(pTaskInfo)); return TSDB_CODE_SUCCESS; } @@ -169,12 +164,12 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { if (ret != TSDB_CODE_SUCCESS) { publishQueryAbortEvent(pTaskInfo, ret); pTaskInfo->code = ret; - qDebug("QInfo:0x%" PRIx64 " query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), + qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); return pTaskInfo->code; } - qDebug("QInfo:0x%" PRIx64 " query task is launched", GET_TASKID(pTaskInfo)); + qDebug("%s execTask is launched", GET_TASKID(pTaskInfo)); bool newgroup = false; publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); @@ -183,66 +178,25 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { st = taosGetTimestampUs(); *pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); - pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); + uint64_t el = (taosGetTimestampUs() - st); + pTaskInfo->cost.elapsedTime += el; + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); if (NULL == *pRes) { *useconds = pTaskInfo->cost.elapsedTime; } - qDebug("QInfo:0x%" PRIx64 " query paused, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d", - GET_TASKID(pTaskInfo), 0, 0L, 0); + int32_t current = (*pRes != NULL)? (*pRes)->info.rows:0; + pTaskInfo->totalRows += current; + + qDebug("%s task suspended, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", + GET_TASKID(pTaskInfo), current, pTaskInfo->totalRows, 0, el/1000.0); atomic_store_64(&pTaskInfo->owner, 0); return pTaskInfo->code; } -int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { - qError("QInfo invalid qhandle"); - return TSDB_CODE_QRY_INVALID_QHANDLE; - } - - *buildRes = false; - if (IS_QUERY_KILLED(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" query is killed, code:0x%08x", pQInfo->qId, pQInfo->code); - return pQInfo->code; - } - - int32_t code = TSDB_CODE_SUCCESS; - - if (tsRetrieveBlockingModel) { - pQInfo->rspContext = pRspContext; - tsem_wait(&pQInfo->ready); - *buildRes = true; - code = pQInfo->code; - } else { - STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - - pthread_mutex_lock(&pQInfo->lock); - - assert(pQInfo->rspContext == NULL); - if (pQInfo->dataReady == QUERY_RESULT_READY) { - *buildRes = true; - qDebug("QInfo:0x%"PRIx64" retrieve result info, rowsize:%d, rows:%d, code:%s", pQInfo->qId, pQueryAttr->resultRowSize, - GET_NUM_OF_RESULTS(pRuntimeEnv), tstrerror(pQInfo->code)); - } else { - *buildRes = false; - qDebug("QInfo:0x%"PRIx64" retrieve req set query return result after paused", pQInfo->qId); - pQInfo->rspContext = pRspContext; - assert(pQInfo->rspContext != NULL); - } - - code = pQInfo->code; - pthread_mutex_unlock(&pQInfo->lock); - } - - return code; -} - void* qGetResultRetrieveMsg(qTaskInfo_t qinfo) { SQInfo* pQInfo = (SQInfo*) qinfo; assert(pQInfo != NULL); @@ -251,18 +205,18 @@ void* qGetResultRetrieveMsg(qTaskInfo_t qinfo) { } int32_t qKillTask(qTaskInfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + if (pTaskInfo == NULL) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - qDebug("QInfo:0x%"PRIx64" query killed", pQInfo->qId); - setQueryKilled(pQInfo); + qDebug("%s execTask killed", GET_TASKID(pTaskInfo)); + setTaskKilled(pTaskInfo); // Wait for the query executing thread being stopped/ // Once the query is stopped, the owner of qHandle will be cleared immediately. - while (pQInfo->owner != 0) { + while (pTaskInfo->owner != 0) { taosMsleep(100); } @@ -270,14 +224,14 @@ int32_t qKillTask(qTaskInfo_t qinfo) { } int32_t qAsyncKillTask(qTaskInfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + if (pTaskInfo == NULL) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - qDebug("QInfo:0x%"PRIx64" query async killed", pQInfo->qId); - setQueryKilled(pQInfo); + qDebug("%s execTask async killed", GET_TASKID(pTaskInfo)); + setTaskKilled(pTaskInfo); return TSDB_CODE_SUCCESS; } @@ -292,145 +246,12 @@ int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { return isTaskKilled(pTaskInfo) || Q_STATUS_EQUAL(pTaskInfo->status, TASK_OVER); } -void qDestroyTask(qTaskInfo_t qHandle) { - SQInfo* pQInfo = (SQInfo*) qHandle; - if (!isValidQInfo(pQInfo)) { - return; - } - - qDebug("QInfo:0x%"PRIx64" query completed", pQInfo->qId); - queryCostStatis(pQInfo); // print the query cost summary - doDestroyTask(pQInfo); -} - -void* qOpenTaskMgmt(int32_t vgId) { - const int32_t refreshHandleInterval = 30; // every 30 seconds, refresh handle pool - - char cacheName[128] = {0}; - sprintf(cacheName, "qhandle_%d", vgId); - - STaskMgmt* pTaskMgmt = calloc(1, sizeof(STaskMgmt)); - if (pTaskMgmt == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - pTaskMgmt->qinfoPool = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshHandleInterval, true, freeqinfoFn, cacheName); - pTaskMgmt->closed = false; - pTaskMgmt->vgId = vgId; - - pthread_mutex_init(&pTaskMgmt->lock, NULL); - - qDebug("vgId:%d, open queryTaskMgmt success", vgId); - return pTaskMgmt; -} - -void qTaskMgmtNotifyClosing(void* pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - STaskMgmt* pQueryMgmt = pQMgmt; - qInfo("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); - - pthread_mutex_lock(&pQueryMgmt->lock); - pQueryMgmt->closed = true; - pthread_mutex_unlock(&pQueryMgmt->lock); - - taosCacheRefresh(pQueryMgmt->qinfoPool, taskMgmtKillTaskFn, NULL); -} - -void qQueryMgmtReOpen(void *pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - STaskMgmt *pQueryMgmt = pQMgmt; - qInfo("vgId:%d, set querymgmt reopen", pQueryMgmt->vgId); - - pthread_mutex_lock(&pQueryMgmt->lock); - pQueryMgmt->closed = false; - pthread_mutex_unlock(&pQueryMgmt->lock); -} - -void qCleanupTaskMgmt(void* pQMgmt) { - if (pQMgmt == NULL) { - return; - } - - STaskMgmt* pQueryMgmt = pQMgmt; - int32_t vgId = pQueryMgmt->vgId; - - assert(pQueryMgmt->closed); - - SCacheObj* pqinfoPool = pQueryMgmt->qinfoPool; - pQueryMgmt->qinfoPool = NULL; - - taosCacheCleanup(pqinfoPool); - pthread_mutex_destroy(&pQueryMgmt->lock); - tfree(pQueryMgmt); - - qDebug("vgId:%d, queryMgmt cleanup completed", vgId); -} - -void** qRegisterTask(void* pMgmt, uint64_t qId, void *qInfo) { - if (pMgmt == NULL) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - STaskMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { - qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into qMgmt, since qMgmt is closed", qId, (void*)qInfo); - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } +void qDestroyTask(qTaskInfo_t qTaskHandle) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*) qTaskHandle; + qDebug("%s execTask completed, numOfRows:%"PRId64, GET_TASKID(pTaskInfo), pTaskInfo->totalRows); - pthread_mutex_lock(&pQueryMgmt->lock); - if (pQueryMgmt->closed) { - pthread_mutex_unlock(&pQueryMgmt->lock); - qError("QInfo:0x%"PRIx64"-%p failed to add qhandle into cache, since qMgmt is colsing", qId, (void*)qInfo); - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } else { - void** handle = taosCachePut(pQueryMgmt->qinfoPool, &qId, sizeof(qId), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE), - (getMaximumIdleDurationSec()*1000)); - pthread_mutex_unlock(&pQueryMgmt->lock); - - return handle; - } -} - -void** qAcquireTask(void* pMgmt, uint64_t _key) { - STaskMgmt *pQueryMgmt = pMgmt; - - if (pQueryMgmt->closed) { - terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; - return NULL; - } - - if (pQueryMgmt->qinfoPool == NULL) { - terrno = TSDB_CODE_QRY_INVALID_QHANDLE; - return NULL; - } - - void** handle = taosCacheAcquireByKey(pQueryMgmt->qinfoPool, &_key, sizeof(_key)); - if (handle == NULL || *handle == NULL) { - terrno = TSDB_CODE_QRY_INVALID_QHANDLE; - return NULL; - } else { - return handle; - } -} - -void** qReleaseTask(void* pMgmt, void* pQInfo, bool freeHandle) { - STaskMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { - return NULL; - } - - taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); - return 0; + queryCostStatis(pTaskInfo); // print the query cost summary + doDestroyTask(pTaskInfo); } #if 0 @@ -444,8 +265,8 @@ int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCo if (pQInfo == NULL || !isValidQInfo(pQInfo)) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - qWarn("QId:0x%"PRIx64" be killed(no memory commit).", pQInfo->qId); - setQueryKilled(pQInfo); + qWarn("%s be killed(no memory commit).", pQInfo->qId); + setTaskKilled(pQInfo); // wait query stop int32_t loop = 0; @@ -461,4 +282,4 @@ int32_t qKillQueryByQId(void* pMgmt, int64_t qId, int32_t waitMs, int32_t waitCo return error; } -#endif \ No newline at end of file +#endif diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index bfd1bce499fc97294982b200e1bb43bd720f33f6..45d1f57e68d6ca05f457ada854214585e7d31693 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2432,10 +2432,6 @@ static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { } bool isTaskKilled(SExecTaskInfo *pTaskInfo) { - if (IS_QUERY_KILLED(pTaskInfo)) { - return true; - } - // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived // abort current query execution. if (pTaskInfo->owner != 0 && ((taosGetTimestampSec() - pTaskInfo->cost.start/1000) > 10*getMaximumIdleDurationSec()) @@ -2444,13 +2440,13 @@ bool isTaskKilled(SExecTaskInfo *pTaskInfo) { assert(pTaskInfo->cost.start != 0); // qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 // ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); - return true; +// return true; } return false; } -void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} +void setTaskKilled(SExecTaskInfo *pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_QUERY_CANCELLED;} //static bool isFixedOutputQuery(STaskAttr* pQueryAttr) { // if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { @@ -4420,33 +4416,32 @@ void calculateOperatorProfResults(SQInfo* pQInfo) { taosArrayDestroy(opStack); } -void queryCostStatis(SQInfo *pQInfo) { - STaskRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - STaskCostInfo *pSummary = &pQInfo->summary; +void queryCostStatis(SExecTaskInfo *pTaskInfo) { + STaskCostInfo *pSummary = &pTaskInfo->cost; - uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); - hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); - pSummary->hashSize = hashSize; +// uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); +// hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); +// pSummary->hashSize = hashSize; // add the merge time pSummary->elapsedTime += pSummary->firstStageMergeTime; - SResultRowPool* p = pQInfo->runtimeEnv.pool; - if (p != NULL) { - pSummary->winInfoSize = getResultRowPoolMemSize(p); - pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); - } else { - pSummary->winInfoSize = 0; - pSummary->numOfTimeWindows = 0; - } - - calculateOperatorProfResults(pQInfo); - - //qDebug("QInfo:0x%"PRIx64" :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " -// "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, -// pQInfo->qId, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, -// pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); +// SResultRowPool* p = pTaskInfo->pool; +// if (p != NULL) { +// pSummary->winInfoSize = getResultRowPoolMemSize(p); +// pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); +// } else { +// pSummary->winInfoSize = 0; +// pSummary->numOfTimeWindows = 0; +// } +// +// calculateOperatorProfResults(pQInfo); + qDebug("%s :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " + "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, + GET_TASKID(pTaskInfo), pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, + pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); +// //qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, // pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); @@ -4995,7 +4990,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { pResultRowInfo->curPos = 0; } - qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + qDebug("%s start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); } @@ -5006,7 +5001,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); // tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + qDebug("%s start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); if (pResultRowInfo->size > 0) { @@ -5151,6 +5146,8 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); if (pExchangeInfo->current >= totalSources) { + qDebug("%s all %"PRIzu" source(s) are exhausted, total rows:%"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, + pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0); return NULL; } @@ -5171,7 +5168,8 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { epSet.port[0] = pSource->addr.epAddr[0].port; tstrncpy(epSet.fqdn[0], pSource->addr.epAddr[0].fqdn, tListLen(epSet.fqdn[0])); - qDebug("QID:0x%" PRIx64 " build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, + int64_t startTs = taosGetTimestampUs(); + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, epSet.fqdn[0], pSource->taskId, pExchangeInfo->current, totalSources); pMsg->header.vgId = htonl(pSource->addr.nodeId); @@ -5182,7 +5180,7 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { // send the fetch remote task result reques pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { - qError("QID:0x%" PRIx64 " prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _error; } @@ -5199,7 +5197,7 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { SRetrieveTableRsp* pRsp = pExchangeInfo->pRsp; if (pRsp->numOfRows == 0) { - qDebug("QID:0x%"PRIx64" vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next", + qDebug("%s vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next", GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1, pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows); @@ -5207,6 +5205,11 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { pExchangeInfo->current += 1; if (pExchangeInfo->current >= totalSources) { + int64_t el = taosGetTimestampUs() - startTs; + pExchangeInfo->totalElapsed += el; + + qDebug("%s all %"PRIzu" sources are exhausted, total rows: %"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, + pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0); return NULL; } else { continue; @@ -5233,21 +5236,24 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { pRes->info.numOfCols = pOperator->numOfOutput; pRes->info.rows = pRsp->numOfRows; + int64_t el = taosGetTimestampUs() - startTs; + pExchangeInfo->totalRows += pRsp->numOfRows; - pExchangeInfo->bytes += pRsp->compLen; + pExchangeInfo->totalSize += pRsp->compLen; pExchangeInfo->rowsOfCurrentSource += pRsp->numOfRows; + pExchangeInfo->totalElapsed += el; if (pRsp->completed == 1) { - qDebug("QID:0x%" PRIx64 " fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64 + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows, pExchangeInfo->bytes, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->current + 1, totalSources); pExchangeInfo->rowsOfCurrentSource = 0; pExchangeInfo->current += 1; } else { - qDebug("QID:0x%" PRIx64 " fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows, pExchangeInfo->bytes); + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows, pExchangeInfo->totalSize); } return pExchangeInfo->pResult; @@ -5294,7 +5300,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); rpcInit.localPort = 0; - rpcInit.label = "TSC"; + rpcInit.label = "EX"; rpcInit.numOfThreads = 1; rpcInit.cfp = processRspMsg; rpcInit.sessions = tsMaxConnections; @@ -7729,33 +7735,37 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t return TSDB_CODE_SUCCESS; } -static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { +static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) { SExecTaskInfo* pTaskInfo = calloc(1, sizeof(SExecTaskInfo)); setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pthread_mutex_init(&pTaskInfo->lock, NULL); pTaskInfo->cost.created = taosGetTimestampMs(); pTaskInfo->id.queryId = queryId; + + char* p = calloc(1, 128); + snprintf(p, 128, "TID:0x%"PRIx64" QID:0x%"PRIx64, taskId, queryId); + pTaskInfo->id.str = strdup(p); + return pTaskInfo; } -static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId); +static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId); -SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* readerHandle, uint64_t queryId) { +SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* readerHandle, uint64_t queryId, uint64_t taskId) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { if (pPhyNode->info.type == OP_TableScan) { SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId); + tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId); return createTableScanOperatorInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); } else if (pPhyNode->info.type == OP_DataBlocksOptScan) { SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId); + tsdbReadHandleT tReaderHandle = doCreateDataReadHandle((STableScanPhyNode*) pPhyNode, readerHandle, (uint64_t) queryId, taskId); return createDataBlocksOptScanInfo(tReaderHandle, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); } else if (pPhyNode->info.type == OP_Exchange) { @@ -7773,13 +7783,13 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask for (int32_t i = 0; i < size; ++i) { SPhyNode* pChildNode = taosArrayGetP(pPhyNode->pChildren, i); - SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, readerHandle, queryId); + SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, readerHandle, queryId, taskId); return createAggregateOperatorInfo(op, pPhyNode->pTargets, pTaskInfo); } } } -static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readerHandle, uint64_t queryId) { +static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, STableGroupInfo* pGroupInfo, void* readerHandle, uint64_t queryId, uint64_t taskId) { STsdbQueryCond cond = {.loadExternalRows = false}; cond.order = pTableScanNode->scan.order; @@ -7798,10 +7808,10 @@ static tsdbReadHandleT createDataReadHandle(STableScanPhyNode* pTableScanNode, S cond.colList[i].colId = pSchema->colId; } - return tsdbQueryTables(readerHandle, &cond, pGroupInfo, queryId, NULL); + return tsdbQueryTables(readerHandle, &cond, pGroupInfo, queryId, taskId); } -static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId) { +static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, void* readerHandle, uint64_t queryId, uint64_t taskId) { int32_t code = 0; STableGroupInfo groupInfo = {0}; @@ -7811,7 +7821,7 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, if (tableType == TSDB_SUPER_TABLE) { code = - tsdbQuerySTableByTagCond(readerHandle, uid, window.skey, NULL, 0, 0, NULL, &groupInfo, NULL, 0, queryId); + tsdbQuerySTableByTagCond(readerHandle, uid, window.skey, NULL, 0, 0, NULL, &groupInfo, NULL, 0, queryId, taskId); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -7821,35 +7831,35 @@ static tsdbReadHandleT doCreateDataReadHandle(STableScanPhyNode* pTableScanNode, SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); - STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = uid}; + STableKeyInfo info = {.lastKey = 0, .uid = uid}; taosArrayPush(pa, &info); taosArrayPush(groupInfo.pGroupList, &pa); } if (groupInfo.numOfTables == 0) { code = 0; - qDebug("no table qualified for query, reqId:0x%"PRIx64, queryId); + qDebug("no table qualified for query, TID:0x%"PRIx64", QID:0x%"PRIx64, taskId, queryId); goto _error; } - return createDataReadHandle(pTableScanNode, &groupInfo, readerHandle, queryId); + return createDataReadHandle(pTableScanNode, &groupInfo, readerHandle, queryId, taskId); _error: terrno = code; return NULL; } -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle, uint64_t taskId) { uint64_t queryId = pPlan->id.queryId; int32_t code = TSDB_CODE_SUCCESS; - *pTaskInfo = createExecTaskInfo(queryId); + *pTaskInfo = createExecTaskInfo(queryId, taskId); if (*pTaskInfo == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _complete; } - (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, readerHandle, queryId); + (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, readerHandle, queryId, taskId); if ((*pTaskInfo)->pRoot == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _complete; @@ -8673,229 +8683,6 @@ FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { return ((SQInfo *)qHandle)->qId == qId; } -SQInfo* createQInfoImpl(SQueryTableReq* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, - SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, SFilterInfo* pFilters, int32_t vgId, - char* sql, uint64_t qId, struct SUdfInfo* pUdfInfo) { - int16_t numOfCols = pQueryMsg->numOfCols; - int16_t numOfOutput = pQueryMsg->numOfOutput; - - SQInfo *pQInfo = (SQInfo *)calloc(1, sizeof(SQInfo)); - if (pQInfo == NULL) { - goto _cleanup_qinfo; - } - - pQInfo->qId = qId; - pQInfo->startExecTs = 0; - - pQInfo->runtimeEnv.pUdfInfo = pUdfInfo; - - // to make sure third party won't overwrite this structure - pQInfo->signature = pQInfo; - STaskAttr* pQueryAttr = &pQInfo->query; - pQInfo->runtimeEnv.pQueryAttr = pQueryAttr; - - pQueryAttr->tableGroupInfo = *pTableGroupInfo; - pQueryAttr->numOfCols = numOfCols; - pQueryAttr->numOfOutput = numOfOutput; - pQueryAttr->limit.limit = pQueryMsg->limit; - pQueryAttr->limit.offset = pQueryMsg->offset; - pQueryAttr->order.order = pQueryMsg->order; - pQueryAttr->order.col.info.colId = pQueryMsg->orderColId; - pQueryAttr->pExpr1 = pExprs; - pQueryAttr->pExpr2 = pSecExprs; - pQueryAttr->numOfExpr2 = pQueryMsg->secondStageOutput; - pQueryAttr->pGroupbyExpr = pGroupbyExpr; - memcpy(&pQueryAttr->interval, &pQueryMsg->interval, sizeof(pQueryAttr->interval)); - pQueryAttr->fillType = pQueryMsg->fillType; - pQueryAttr->numOfTags = pQueryMsg->numOfTags; - pQueryAttr->tagColList = pTagCols; - pQueryAttr->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit; - pQueryAttr->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX; -// pQueryAttr->sw = pQueryMsg->sw; - pQueryAttr->vgId = vgId; - - pQueryAttr->stableQuery = pQueryMsg->stableQuery; - pQueryAttr->topBotQuery = pQueryMsg->topBotQuery; - pQueryAttr->groupbyColumn = pQueryMsg->groupbyColumn; - pQueryAttr->hasTagResults = pQueryMsg->hasTagResults; - pQueryAttr->timeWindowInterpo = pQueryMsg->timeWindowInterpo; - pQueryAttr->queryBlockDist = pQueryMsg->queryBlockDist; - pQueryAttr->stabledev = pQueryMsg->stabledev; - pQueryAttr->tsCompQuery = pQueryMsg->tsCompQuery; - pQueryAttr->simpleAgg = pQueryMsg->simpleAgg; - pQueryAttr->pointInterpQuery = pQueryMsg->pointInterpQuery; - pQueryAttr->needReverseScan = pQueryMsg->needReverseScan; - pQueryAttr->stateWindow = pQueryMsg->stateWindow; - pQueryAttr->vgId = vgId; -// pQueryAttr->pFilters = pFilters; - - pQueryAttr->tableCols = calloc(numOfCols, sizeof(SSingleColumnFilterInfo)); - if (pQueryAttr->tableCols == NULL) { - goto _cleanup; - } - - pQueryAttr->srcRowSize = 0; - pQueryAttr->maxTableColumnWidth = 0; - for (int16_t i = 0; i < numOfCols; ++i) { - pQueryAttr->tableCols[i] = pQueryMsg->tableCols[i]; -// pQueryAttr->tableCols[i].flist.filterInfo = tFilterInfoDup(pQueryMsg->tableCols[i].flist.filterInfo, pQueryAttr->tableCols[i].flist.numOfFilters); - - pQueryAttr->srcRowSize += pQueryAttr->tableCols[i].bytes; - if (pQueryAttr->maxTableColumnWidth < pQueryAttr->tableCols[i].bytes) { - pQueryAttr->maxTableColumnWidth = pQueryAttr->tableCols[i].bytes; - } - } - - for (int16_t col = 0; col < numOfOutput; ++col) { - assert(pExprs[col].base.resSchema.bytes > 0); - pQueryAttr->resultRowSize += pExprs[col].base.resSchema.bytes; - - // keep the tag length - if (TSDB_COL_IS_TAG(pExprs[col].base.pColumns->flag)) { - pQueryAttr->tagLen += pExprs[col].base.resSchema.bytes; - } - -// if (pExprs[col].base.flist.filterInfo) { -// ++pQueryAttr->havingNum; -// } - } - - doUpdateExprColumnIndex(pQueryAttr); - - if (pSecExprs != NULL) { - int32_t resultRowSize = 0; - - // calculate the result row size - for (int16_t col = 0; col < pQueryAttr->numOfExpr2; ++col) { - assert(pSecExprs[col].base.resSchema.bytes > 0); - resultRowSize += pSecExprs[col].base.resSchema.bytes; - } - - if (resultRowSize > pQueryAttr->resultRowSize) { - pQueryAttr->resultRowSize = resultRowSize; - } - } - - if (pQueryAttr->fillType != TSDB_FILL_NONE) { - pQueryAttr->fillVal = malloc(sizeof(int64_t) * pQueryAttr->numOfOutput); - if (pQueryAttr->fillVal == NULL) { - goto _cleanup; - } - - // the first column is the timestamp - memcpy(pQueryAttr->fillVal, (char *)pQueryMsg->fillVal, pQueryAttr->numOfOutput * sizeof(int64_t)); - } - - size_t numOfGroups = 0; - if (pTableGroupInfo->pGroupList != NULL) { - numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); - STableGroupInfo* pTableqinfo = &pQInfo->runtimeEnv.tableqinfoGroupInfo; - - pTableqinfo->pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); - pTableqinfo->numOfTables = pTableGroupInfo->numOfTables; - pTableqinfo->map = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - } - - pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); - if (pQInfo->pBuf == NULL) { - goto _cleanup; - } - - pQInfo->dataReady = QUERY_RESULT_NOT_READY; - pQInfo->rspContext = NULL; - pQInfo->sql = sql; - pthread_mutex_init(&pQInfo->lock, NULL); - tsem_init(&pQInfo->ready, 0, 0); - - pQueryAttr->window = pQueryMsg->window; - updateDataCheckOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery); - - STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - STimeWindow window = pQueryAttr->window; - - int32_t index = 0; - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray* pa = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t s = taosArrayGetSize(pa); - SArray* p1 = taosArrayInit(s, POINTER_BYTES); - if (p1 == NULL) { - goto _cleanup; - } - - taosArrayPush(pRuntimeEnv->tableqinfoGroupInfo.pGroupList, &p1); - - for(int32_t j = 0; j < s; ++j) { -// STableKeyInfo* info = taosArrayGet(pa, j); -// window.skey = info->lastKey; -// -// void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); -// STableQueryInfo* item = createTableQueryInfo(pQueryAttr, info->pTable, pQueryAttr->groupbyColumn, window, buf); -// if (item == NULL) { -// goto _cleanup; -// } -// -// item->groupIndex = i; -// taosArrayPush(p1, &item); - -// STableId* id = TSDB_TABLEID(info->pTable); -// taosHashPut(pRuntimeEnv->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); -// index += 1; - } - } - - colIdCheck(pQueryAttr, pQInfo->qId); - -// int32_t functionId = getExprFunctionId(&pExpr[0]); -// pQInfo->query.queryBlockDist = (functionId == FUNCTION_BLKINFO); - - //qDebug("qmsg:%p vgId:%d, QInfo:0x%" PRIx64 "-%p created", pQueryMsg, pQInfo->query.vgId, pQInfo->qId, pQInfo); - return pQInfo; - -_cleanup_qinfo: -// tsdbDestroyTableGroup(pTableGroupInfo); - - if (pGroupbyExpr != NULL) { - taosArrayDestroy(pGroupbyExpr->columnInfo); - free(pGroupbyExpr); - } - - tfree(pTagCols); - for (int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = &pExprs[i]; - if (pExprInfo->pExpr != NULL) { - tExprTreeDestroy(pExprInfo->pExpr, NULL); - pExprInfo->pExpr = NULL; - } - -// if (pExprInfo->base.flist.filterInfo) { -// freeColumnFilterInfo(pExprInfo->base.flist.filterInfo, pExprInfo->base.flist.numOfFilters); -// } - } - - tfree(pExprs); - -// filterFreeInfo(pFilters); - -_cleanup: - doDestroyTask(pQInfo); - return NULL; -} - -bool isValidQInfo(void *param) { - SQInfo *pQInfo = (SQInfo *)param; - if (pQInfo == NULL) { - return false; - } - - /* - * pQInfo->signature may be changed by another thread, so we assign value of signature - * into local variable, then compare by using local variable - */ - uint64_t sig = (uint64_t)pQInfo->signature; - return (sig == (uint64_t)pQInfo); -} - int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start, int32_t prevResultLen, void* merger) { int32_t code = TSDB_CODE_SUCCESS; @@ -8957,7 +8744,7 @@ int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* _error: // table query ref will be decrease during error handling - doDestroyTask(pQInfo); +// doDestroyTask(pQInfo); return code; } @@ -9038,113 +8825,16 @@ void* freeColumnInfo(SColumnInfo* pColumnInfo, int32_t numOfCols) { return NULL; } -void doDestroyTask(SQInfo *pQInfo) { - if (!isValidQInfo(pQInfo)) { - return; - } - - //qDebug("QInfo:0x%"PRIx64" start to free QInfo", pQInfo->qId); - - STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables); - - doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo); - teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - - STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - freeQueryAttr(pQueryAttr); - -// tsdbDestroyTableGroup(&pQueryAttr->tableGroupInfo); - - tfree(pQInfo->pBuf); - tfree(pQInfo->sql); - - taosArrayDestroy(pQInfo->summary.queryProfEvents); - taosHashCleanup(pQInfo->summary.operatorProfResults); - - taosArrayDestroy(pRuntimeEnv->groupResInfo.pRows); - pQInfo->signature = 0; - - //qDebug("QInfo:0x%"PRIx64" QInfo is freed", pQInfo->qId); - - tfree(pQInfo); -} - -int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen) { - // the remained number of retrieved rows, not the interpolated result - STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - STaskAttr *pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; - - // load data from file to msg buffer - if (pQueryAttr->tsCompQuery) { - SColumnInfoData* pColInfoData = taosArrayGet(pRuntimeEnv->outputBuf->pDataBlock, 0); - FILE *f = *(FILE **)pColInfoData->pData; // TODO refactor - - // make sure file exist - if (f) { - off_t s = lseek(fileno(f), 0, SEEK_END); - assert(s == pRuntimeEnv->outputBuf->info.rows); - - //qDebug("QInfo:0x%"PRIx64" ts comp data return, file:%p, size:%"PRId64, pQInfo->qId, f, (uint64_t)s); - if (fseek(f, 0, SEEK_SET) >= 0) { - size_t sz = fread(data, 1, s, f); - if(sz < s) { // todo handle error - //qError("fread(f:%p,%d) failed, rsize:%" PRId64 ", expect size:%" PRId64, f, fileno(f), (uint64_t)sz, (uint64_t)s); - assert(0); - } - } else { - UNUSED(s); - //qError("fseek(f:%p,%d) failed, error:%s", f, fileno(f), strerror(errno)); - assert(0); - } - - // dump error info - if (s <= (sizeof(STSBufFileHeader) + sizeof(STSGroupBlockInfo) + 6 * sizeof(int32_t))) { -// qDump(data, s); - assert(0); - } - - fclose(f); - *(FILE **)pColInfoData->pData = NULL; - } - - // all data returned, set query over - if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)) { -// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); - } - } else { - doCopyQueryResultToMsg(pQInfo, (int32_t)pRuntimeEnv->outputBuf->info.rows, data, compressed, compLen); - } - - //qDebug("QInfo:0x%"PRIx64" current numOfRes rows:%d, total:%" PRId64, pQInfo->qId, -// pRuntimeEnv->outputBuf->info.rows, pRuntimeEnv->resultInfo.total); - - if (pQueryAttr->limit.limit > 0 && pQueryAttr->limit.limit == pRuntimeEnv->resultInfo.total) { - //qDebug("QInfo:0x%"PRIx64" results limitation reached, limitation:%"PRId64, pQInfo->qId, pQueryAttr->limit.limit); -// setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); - } - - return TSDB_CODE_SUCCESS; -} - -bool doBuildResCheck(SQInfo* pQInfo) { - bool buildRes = false; - - pthread_mutex_lock(&pQInfo->lock); - - pQInfo->dataReady = QUERY_RESULT_READY; - buildRes = needBuildResAfterQueryComplete(pQInfo); - - // clear qhandle owner, it must be in the secure area. other thread may run ahead before current, after it is - // put into task to be executed. - assert(pQInfo->owner == taosGetSelfPthreadId()); - pQInfo->owner = 0; +void doDestroyTask(SExecTaskInfo *pTaskInfo) { + doDestroyTableQueryInfo(&pTaskInfo->tableqinfoGroupInfo); +// taosArrayDestroy(pTaskInfo->summary.queryProfEvents); +// taosHashCleanup(pTaskInfo->summary.operatorProfResults); - pthread_mutex_unlock(&pQInfo->lock); + tfree(pTaskInfo->sql); + tfree(pTaskInfo->id.str); + qDebug("%s execTask is freed", GET_TASKID(pTaskInfo)); - // used in retrieve blocking model. - tsem_post(&pQInfo->ready); - return buildRes; + tfree(pTaskInfo); } static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type, int16_t bytes) { diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 5e0b641b285927c7b02644bce2f2ae831ff2dcb8..1814e1724b94c5f1e6397789b9fcee1d77af62bb 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -219,7 +219,7 @@ TEST(testCase, build_executor_tree_Test) { SExecTaskInfo* pTaskInfo = nullptr; DataSinkHandle sinkHandle = nullptr; - int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo, &sinkHandle); + int32_t code = qCreateExecTask((void*) 1, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); } #pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h index cd84222c65da90672987ef4acb11ad4764a82ff6..f53c5ea279f134d09b9a48ec40bcaa3fe6a934b1 100644 --- a/source/libs/parser/inc/dataBlockMgt.h +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -171,6 +171,8 @@ int32_t boundIdxCompar(const void *lhs, const void *rhs); void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t numOfCols); void destroyBoundColumnInfo(SParsedDataColInfo* pColList); void destroyBlockArrayList(SArray* pDataBlockList); +void destroyBlockHashmap(SHashObj* pDataBlockHash); + int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen); int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 6e91ad997cc5454099634c8f231b5c4f5cf2e6b9..410e4fb1875e5fc76966b1af5e1c08f3773e602e 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -65,7 +65,7 @@ program ::= cmd. {} //////////////////////////////////THE SHOW STATEMENT/////////////////////////////////////////// cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);} cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} -cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);} +cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNC, 0, 0);} cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);} cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);} diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 563443d3c3c8f80d6c454e3c87047f5451048ae7..0a2f2b20f2c77170d04b2c9cf7705dd2a1129b32 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -213,10 +213,11 @@ SQueryStmtInfo *createQueryInfo() { pQueryInfo->slimit.limit = -1; pQueryInfo->slimit.offset = 0; - pQueryInfo->pDownstream = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->pDownstream = taosArrayInit(4, POINTER_BYTES); pQueryInfo->window = TSWINDOW_INITIALIZER; pQueryInfo->exprList = calloc(10, POINTER_BYTES); + for(int32_t i = 0; i < 10; ++i) { pQueryInfo->exprList[i] = taosArrayInit(4, POINTER_BYTES); } @@ -232,7 +233,8 @@ static void destroyQueryInfoImpl(SQueryStmtInfo* pQueryInfo) { cleanupFieldInfo(&pQueryInfo->fieldsInfo); dropAllExprInfo(pQueryInfo->exprList, 10); - pQueryInfo->exprList = NULL; + + tfree(pQueryInfo->exprList); columnListDestroy(pQueryInfo->colList); pQueryInfo->colList = NULL; @@ -258,10 +260,10 @@ void destroyQueryInfo(SQueryStmtInfo* pQueryInfo) { size_t numOfUpstream = taosArrayGetSize(pQueryInfo->pDownstream); for (int32_t i = 0; i < numOfUpstream; ++i) { - SQueryStmtInfo* pUpQueryInfo = taosArrayGetP(pQueryInfo->pDownstream, i); - destroyQueryInfoImpl(pUpQueryInfo); - clearAllTableMetaInfo(pUpQueryInfo, false, 0); - tfree(pUpQueryInfo); + SQueryStmtInfo* pDownstream = taosArrayGetP(pQueryInfo->pDownstream, i); + destroyQueryInfoImpl(pDownstream); + clearAllTableMetaInfo(pDownstream, false, 0); + tfree(pDownstream); } destroyQueryInfoImpl(pQueryInfo); @@ -1395,6 +1397,13 @@ int32_t validateFillNode(SQueryStmtInfo *pQueryInfo, SSqlNode* pSqlNode, SMsgBuf static void pushDownAggFuncExprInfo(SQueryStmtInfo* pQueryInfo); static void addColumnNodeFromLowerLevel(SQueryStmtInfo* pQueryInfo); +static void freeItemHelper(void* pItem) { + void** p = pItem; + if (*p != NULL) { + tfree(*p); + } +} + int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf) { assert(pSqlNode != NULL && (pSqlNode->from == NULL || taosArrayGetSize(pSqlNode->from->list) > 0)); @@ -1590,7 +1599,10 @@ int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* SArray* functionList = extractFunctionList(pQueryInfo->exprList[i]); extractFunctionDesc(functionList, &pQueryInfo->info); - if ((code = checkForInvalidExpr(pQueryInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) { + code = checkForInvalidExpr(pQueryInfo, pMsgBuf); + taosArrayDestroyEx(functionList, freeItemHelper); + + if (code != TSDB_CODE_SUCCESS) { return code; } } @@ -2902,6 +2914,8 @@ int32_t doAddOneProjectCol(SQueryStmtInfo* pQueryInfo, int32_t outputColIndex, S } pQueryInfo->info.projectionQuery = true; + + taosArrayDestroy(pColumnList); return TSDB_CODE_SUCCESS; } @@ -3983,5 +3997,9 @@ int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtI validateSqlNode(p, pQueryInfo, &buf); } + taosArrayDestroy(data.pTableMeta); + taosArrayDestroy(req.pUdf); + taosArrayDestroy(req.pTableName); + return code; } diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index bc4eae7ae99aa022ad3be554a210014968155673..ece3d5f5eb51699761a2e1d7f1684dcb4c8de6bc 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -249,7 +249,7 @@ static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlo } } -void destroyDataBlock(STableDataBlocks* pDataBlock) { +static void destroyDataBlock(STableDataBlocks* pDataBlock) { if (pDataBlock == NULL) { return; } @@ -273,12 +273,29 @@ void destroyBlockArrayList(SArray* pDataBlockList) { size_t size = taosArrayGetSize(pDataBlockList); for (int32_t i = 0; i < size; i++) { - destroyDataBlock(taosArrayGetP(pDataBlockList, i)); + void* p = taosArrayGetP(pDataBlockList, i); + destroyDataBlock(p); } taosArrayDestroy(pDataBlockList); } +void destroyBlockHashmap(SHashObj* pDataBlockHash) { + if (pDataBlockHash == NULL) { + return; + } + + void** p1 = taosHashIterate(pDataBlockHash, NULL); + while (p1) { + STableDataBlocks* pBlocks = *p1; + destroyDataBlock(pBlocks); + + p1 = taosHashIterate(pDataBlockHash, p1); + } + + taosHashCleanup(pDataBlockHash); +} + // data block is disordered, sort it in ascending order void sortRemoveDataBlockDupRowsRaw(STableDataBlocks *dataBuf) { SSubmitBlk *pBlocks = (SSubmitBlk *)dataBuf->pData; @@ -490,7 +507,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t } // the maximum expanded size in byte when a row-wise data is converted to SDataRow format - int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; + int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0; int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize + sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index a031199992fa723555c9e654d96b655c3c7aa58c..cb7df824b0206a47e6f3b07537a14c4bd5c796db 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -525,10 +525,28 @@ static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { tdDestroyKVRowBuilder(&pCxt->tagsBuilder); } +static void destroyDataBlock(STableDataBlocks* pDataBlock) { + if (pDataBlock == NULL) { + return; + } + + tfree(pDataBlock->pData); + if (!pDataBlock->cloned) { + // free the refcount for metermeta + if (pDataBlock->pTableMeta != NULL) { + tfree(pDataBlock->pTableMeta); + } + + destroyBoundColumnInfo(&pDataBlock->boundColumnInfo); + } + tfree(pDataBlock); +} + static void destroyInsertParseContext(SInsertParseContext* pCxt) { destroyInsertParseContextForTable(pCxt); taosHashCleanup(pCxt->pVgroupsHashObj); - taosHashCleanup(pCxt->pTableBlockHashObj); + + destroyBlockHashmap(pCxt->pTableBlockHashObj); destroyBlockArrayList(pCxt->pTableDataBlocks); destroyBlockArrayList(pCxt->pVgDataBlocks); } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 4271aae451e9f2c545910af84e20346a0cd7f0dd..c98f787f0b9c862362582553ac7c2875e3c706d0 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -248,10 +248,15 @@ void qDestroyQuery(SQueryNode* pQueryNode) { if (NULL == pQueryNode) { return; } - if (nodeType(pQueryNode) == TSDB_SQL_INSERT || nodeType(pQueryNode) == TSDB_SQL_CREATE_TABLE) { + + int32_t type = nodeType(pQueryNode); + if (type == TSDB_SQL_INSERT || type == TSDB_SQL_CREATE_TABLE) { SVnodeModifOpStmtInfo* pModifInfo = (SVnodeModifOpStmtInfo*)pQueryNode; taosArrayDestroy(pModifInfo->pDataBlocks); - } - tfree(pQueryNode); + tfree(pQueryNode); + } else if (type == TSDB_SQL_SELECT) { + SQueryStmtInfo* pQueryStmtInfo = (SQueryStmtInfo*) pQueryNode; + destroyQueryInfo(pQueryStmtInfo); + } } diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index b8545b748636bb5672ebcff37157cddbc8214c70..a87e138ed0f201ce098b3f2fc132d6cf4d0a63c1 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -732,18 +732,8 @@ void cleanupFieldInfo(SFieldInfo* pFieldInfo) { return; } - if (pFieldInfo->internalField != NULL) { - size_t num = taosArrayGetSize(pFieldInfo->internalField); - for (int32_t i = 0; i < num; ++i) { -// SInternalField* pfield = taosArrayGet(pFieldInfo->internalField, i); -// if (pfield->pExpr != NULL && pfield->pExpr->pExpr != NULL) { -// sqlExprDestroy(pfield->pExpr); -// } - } - } - taosArrayDestroy(pFieldInfo->internalField); -// tfree(pFieldInfo->final); + tfree(pFieldInfo->final); memset(pFieldInfo, 0, sizeof(SFieldInfo)); } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index 1aa8836cb2c92de31520490da9590e4dcee6d65c..9a2ca2da98399455653c24074368b9df6471e20d 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -191,10 +191,12 @@ void destroyExprInfo(SExprInfo* pExprInfo) { for(int32_t i = 0; i < pExprInfo->base.numOfParams; ++i) { taosVariantDestroy(&pExprInfo->base.param[i]); } + + tfree(pExprInfo->base.pColumns); tfree(pExprInfo); } -static void dropOneLevelExprInfo(SArray* pExprInfo) { +void dropOneLevelExprInfo(SArray* pExprInfo) { size_t size = taosArrayGetSize(pExprInfo); for (int32_t i = 0; i < size; ++i) { @@ -239,6 +241,9 @@ void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { #endif dst->pExpr = exprdup(src->pExpr); + dst->base.pColumns = calloc(src->base.numOfCols, sizeof(SColumn)); + memcpy(dst->base.pColumns, src->base.pColumns, sizeof(SColumn) * src->base.numOfCols); + memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param)); for (int32_t j = 0; j < src->base.numOfParams; ++j) { taosVariantAssign(&dst->base.param[j], &src->base.param[j]); diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index a6537998f7697d0cf740f9f6924223a0031e20b4..6f63feb3c40f38cf5a18d5cd5f28b8eeb7ac4045 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -2232,7 +2232,7 @@ static void yy_reduce( { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);} break; case 3: /* cmd ::= SHOW FUNCTIONS */ -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);} +{ setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNC, 0, 0);} break; case 4: /* cmd ::= SHOW MNODES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);} diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index e817b764d5ff1aab8372718649722f65a097a654..de62f4a2ef8a505fd17340a873e7e1fce6d098fe 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -265,7 +265,6 @@ static SQueryPlanNode* doCreateQueryPlanForSingleTableImpl(const SQueryStmtInfo* } else { // here we can push down the projection to tablescan operator. pNode->numOfExpr = num; - pNode->pExpr = taosArrayInit(num, POINTER_BYTES); taosArrayAddAll(pNode->pExpr, p); } } @@ -357,7 +356,6 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { SArray* exprList = taosArrayInit(4, POINTER_BYTES); if (copyExprInfoList(exprList, pQueryInfo->exprList[0], uid, true) != 0) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; -// dropAllExprInfo(exprList); exit(-1); } @@ -373,7 +371,6 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { // 4. add the projection query node SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, &info, exprList, tableColumnList); columnListDestroy(tableColumnList); -// dropAllExprInfo(exprList); taosArrayPush(pDownstream, &pNode); } @@ -398,7 +395,8 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) { } static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { - if (pQueryNode->info.type == QNODE_MODIFY) { + int32_t type = nodeType(pQueryNode); + if (type == QNODE_MODIFY) { SDataPayloadInfo* pInfo = pQueryNode->pExtInfo; size_t size = taosArrayGetSize(pInfo->payload); @@ -410,10 +408,16 @@ static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { taosArrayDestroy(pInfo->payload); } + if (type == QNODE_STREAMSCAN || type == QNODE_TABLESCAN) { + SQueryTableInfo* pQueryTableInfo = pQueryNode->pExtInfo; + tfree(pQueryTableInfo->tableName); + } + + taosArrayDestroy(pQueryNode->pExpr); + tfree(pQueryNode->pExtInfo); tfree(pQueryNode->pSchema); tfree(pQueryNode->info.name); -// dropAllExprInfo(pQueryNode->pExpr); if (pQueryNode->pChildren != NULL) { int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pChildren); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 9288c240d0309ddc45b5d991a0192f6f291ec8e1..12c0d0780b677ebfc6457bdbf561dbba05cbc968 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -155,6 +155,16 @@ static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t si return node; } +static void cleanupPhyNode(SPhyNode* pPhyNode) { + if (pPhyNode == NULL) { + return; + } + + dropOneLevelExprInfo(pPhyNode->pTargets); + tfree(pPhyNode->targetSchema.pSchema); + tfree(pPhyNode); +} + static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) { SScanPhyNode* node = (SScanPhyNode*) initPhyNode(pPlanNode, type, size); @@ -445,3 +455,29 @@ void setExchangSourceNode(uint64_t templateId, SDownstreamSource *pSource, SPhyN void setSubplanExecutionNode(SSubplan* subplan, uint64_t templateId, SDownstreamSource* pSource) { setExchangSourceNode(templateId, pSource, subplan->pNode); } + +static void destroyDataSinkNode(SDataSink* pSinkNode) { + if (pSinkNode == NULL) { + return; + } + + if (nodeType(pSinkNode) == DSINK_Dispatch) { + SDataDispatcher* pDdSink = (SDataDispatcher*)pSinkNode; + tfree(pDdSink->sink.schema.pSchema); + } + + tfree(pSinkNode); +} + +void qDestroySubplan(SSubplan* pSubplan) { + if (pSubplan == NULL) { + return; + } + + taosArrayDestroy(pSubplan->pChildren); + taosArrayDestroy(pSubplan->pParents); + destroyDataSinkNode(pSubplan->pDataSink); + cleanupPhyNode(pSubplan->pNode); + + tfree(pSubplan); +} diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index cf54fdec85c88ecc928e2f263a681ada6f514d07..4fbec534f6f9546a8985bc1c7b2da7ce728f8be3 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -87,6 +87,7 @@ static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson f static const char* jkPnodeType = "Type"; static int32_t getPnodeTypeSize(cJSON* json) { switch (getNumber(json, jkPnodeType)) { + case OP_StreamScan: case OP_TableScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: @@ -869,6 +870,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { SPhyNode* phyNode = (SPhyNode*)obj; switch (phyNode->info.type) { case OP_TableScan: + case OP_StreamScan: case OP_DataBlocksOptScan: case OP_TableSeqScan: return tableScanNodeFromJson(json, obj); @@ -1121,6 +1123,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); + cJSON_Delete(json); + // printf("====Physical plan:====\n"); // printf("%s\n", *str); *len = strlen(*str) + 1; @@ -1187,14 +1191,14 @@ SQueryDag* qJsonToDag(const cJSON* pRoot) { if(pDag == NULL) { return NULL; } - pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "numOfSubplans")); - pDag->queryId = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "queryId")); + pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "Number")); + pDag->queryId = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "QueryId")); pDag->pSubplans = taosArrayInit(0, sizeof(SArray)); if (pDag->pSubplans == NULL) { free(pDag); return NULL; } - cJSON* pLevels = cJSON_GetObjectItem(pRoot, "pSubplans"); + cJSON* pLevels = cJSON_GetObjectItem(pRoot, "Subplans"); int level = cJSON_GetArraySize(pLevels); for(int i = 0; i < level; i++) { SArray* plansOneLevel = taosArrayInit(0, sizeof(void*)); diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 2fc4c8ea3e1348c7568399176622b6b8bdb75455..d546925c5f20f4d0cda1850364278d9b6ac913e7 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -18,25 +18,6 @@ static void extractResSchema(struct SQueryDag* const* pDag, SSchema** pResSchema, int32_t* numOfCols); -static void destroyDataSinkNode(SDataSink* pSinkNode) { - if (pSinkNode == NULL) { - return; - } - tfree(pSinkNode); -} - -void qDestroySubplan(SSubplan* pSubplan) { - if (pSubplan == NULL) { - return; - } - - taosArrayDestroy(pSubplan->pChildren); - taosArrayDestroy(pSubplan->pParents); - destroyDataSinkNode(pSubplan->pDataSink); - // todo destroy pNode - tfree(pSubplan); -} - void qDestroyQueryDag(struct SQueryDag* pDag) { if (pDag == NULL) { return; @@ -51,6 +32,7 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { SSubplan* pSubplan = taosArrayGetP(pa, j); qDestroySubplan(pSubplan); } + taosArrayDestroy(pa); } diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 83c7a4208e3da2cbd8b2c58040afd1520bb01cc5..d07e20d5c14762ddb68b8c15fa93963a14043115 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -173,17 +173,17 @@ typedef struct SQWorkerMgmt { #define QW_SCH_ELOG(param, ...) qError("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) #define QW_SCH_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64" " param, mgmt, sId, __VA_ARGS__) -#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_ELOG(param, ...) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_WLOG(param, ...) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) +#define QW_TASK_DLOG(param, ...) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId, __VA_ARGS__) -#define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) -#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) -#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_ELOG_E(param) qError("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_WLOG_E(param) qWarn("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) +#define QW_TASK_DLOG_E(param) qDebug("QW:%p QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, qId, tId) -#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_ELOG(param, ...) qError("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_SCH_TASK_WLOG(param, ...) qWarn("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) -#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) +#define QW_SCH_TASK_DLOG(param, ...) qDebug("QW:%p SID:%"PRIx64",QID:0x%"PRIx64",TID:0x%"PRIx64" " param, mgmt, sId, qId, tId, __VA_ARGS__) #define QW_LOCK_DEBUG(...) do { if (gQWDebug.lockDebug) { qDebug(__VA_ARGS__); } } while (0) diff --git a/source/libs/qworker/inc/qworkerMsg.h b/source/libs/qworker/inc/qworkerMsg.h index 7735e1a1eebd4e329f0df807d7269e7754e4c9ee..51f55d238f33f3c109ec51e3df6f845f5f5aabac 100644 --- a/source/libs/qworker/inc/qworkerMsg.h +++ b/source/libs/qworker/inc/qworkerMsg.h @@ -32,7 +32,7 @@ int32_t qwProcessDrop(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t t int32_t qwBuildAndSendDropRsp(void *connection, int32_t code); int32_t qwBuildAndSendCancelRsp(SRpcMsg *pMsg, int32_t code); int32_t qwBuildAndSendFetchRsp(void *connection, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); -void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len); +void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete); int32_t qwBuildAndSendCQueryMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection); int32_t qwBuildAndSendSchSinkMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, void *connection); int32_t qwBuildAndSendReadyRsp(void *connection, int32_t code); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index b53fd9ff37e68faf8e76ff66aef9ab515f20eb26..c28e30333c15604ad872f71b9f53a13b022fa8ac 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -456,7 +456,7 @@ _return: QW_RET(code); } -int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { +int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { int32_t code = 0; bool qcontinue = true; SSDataBlock* pRes = NULL; @@ -467,11 +467,11 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { DataSinkHandle sinkHandle = ctx->sinkHandle; while (true) { - QW_TASK_DLOG("start to execTask in executor, loopIdx:%d", i++); + QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); code = qExecTask(*taskHandle, &pRes, &useconds); if (code) { - QW_TASK_ELOG("qExecTask failed, code:%x", code); + QW_TASK_ELOG("qExecTask failed, code:%s", tstrerror(code)); QW_ERR_JRET(code); } @@ -485,6 +485,10 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { if (TASK_TYPE_TEMP == ctx->taskType) { qwFreeTaskHandle(QW_FPARAMS(), taskHandle); } + + if (queryEnd) { + *queryEnd = true; + } break; } @@ -492,7 +496,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL}; code = dsPutDataBlock(sinkHandle, &inputData, &qcontinue); if (code) { - QW_TASK_ELOG("dsPutDataBlock failed, code:%x", code); + QW_TASK_ELOG("dsPutDataBlock failed, code:%s", tstrerror(code)); QW_ERR_JRET(code); } @@ -587,12 +591,7 @@ int32_t qwGetResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void QW_ERR_RET(code); } - queryEnd = pOutput->queryEnd; - pOutput->queryEnd = false; - - if (DS_BUF_EMPTY == pOutput->bufStatus && queryEnd) { - pOutput->queryEnd = true; - + if (DS_BUF_EMPTY == pOutput->bufStatus && pOutput->queryEnd) { QW_SCH_TASK_DLOG("task all fetched, status:%d", JOB_TASK_STATUS_SUCCEED); QW_ERR_RET(qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCCEED)); } @@ -974,7 +973,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { QW_ERR_JRET(code); } - code = qCreateExecTask(qwMsg->node, 0, (struct SSubplan *)plan, &pTaskInfo, &sinkHandle); + code = qCreateExecTask(qwMsg->node, 0, tId, (struct SSubplan *)plan, &pTaskInfo, &sinkHandle); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%s", tstrerror(code)); QW_ERR_JRET(code); @@ -996,7 +995,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { atomic_store_ptr(&ctx->sinkHandle, sinkHandle); if (pTaskInfo && sinkHandle) { - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx)); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, NULL)); } _return: @@ -1083,6 +1082,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { SQWPhaseOutput output = {0}; void *rsp = NULL; int32_t dataLen = 0; + bool queryEnd = false; do { QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_CQUERY, &input, &output)); @@ -1102,7 +1102,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { DataSinkHandle sinkHandle = ctx->sinkHandle; - QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx)); + QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryEnd)); if (QW_IS_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { SOutputData sOutput = {0}; @@ -1114,13 +1114,10 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { // RC WARNING atomic_store_8(&ctx->queryContinue, 1); } - - if (sOutput.queryEnd) { - needStop = true; - } if (rsp) { - qwBuildFetchRsp(rsp, &sOutput, dataLen); + bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); @@ -1131,6 +1128,10 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } } + if (queryEnd) { + needStop = true; + } + _return: if (NULL == ctx) { @@ -1196,7 +1197,8 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { if (NULL == rsp) { QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); } else { - qwBuildFetchRsp(rsp, &sOutput, dataLen); + bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); + qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); } if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index d11bee6dce22454646d5080cb87314c8beb73685..56c882f404a36b6b070e0df8b79c3b88af56890d 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -26,11 +26,11 @@ int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { return TSDB_CODE_SUCCESS; } -void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len) { +void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; rsp->useconds = htobe64(input->useconds); - rsp->completed = input->queryEnd; + rsp->completed = qComplete; rsp->precision = input->precision; rsp->compressed = input->compressed; rsp->compLen = htonl(len); @@ -258,12 +258,12 @@ int32_t qwBuildAndSendCQueryMsg(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg); if (TSDB_CODE_SUCCESS != code) { - QW_SCH_TASK_ELOG("put query continue msg to queue failed, code:%x", code); + QW_SCH_TASK_ELOG("put query continue msg to queue failed, vgId:%d, code:%s", mgmt->nodeId, tstrerror(code)); rpcFreeCont(req); QW_ERR_RET(code); } - QW_SCH_TASK_DLOG("put query continue msg to query queue, vgId:%d", mgmt->nodeId); + QW_SCH_TASK_DLOG("put task continue exec msg to query queue, vgId:%d", mgmt->nodeId); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index f90fdd6e11cda3d23ff68413233201c7953d45ed..934b222f641fe4e0b2fdffaf0531ae9f02c3c849 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -1482,22 +1482,23 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { } int32_t msgSize = sizeof(SSubQueryMsg) + msgLen; - msg = calloc(1, msgSize); if (NULL == msg) { qError("calloc %d failed", msgSize); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SSubQueryMsg *pMsg = (SSubQueryMsg*) msg; + SSubQueryMsg* pMsg = calloc(1, msgSize); + /*SSubQueryMsg *pMsg = (SSubQueryMsg*) msg;*/ + memcpy(pMsg->msg, msg, msgLen); - pMsg->header.vgId = htonl(tInfo.addr.nodeId); + pMsg->header.vgId = tInfo.addr.nodeId; pMsg->sId = schMgmt.sId; pMsg->queryId = plan->id.queryId; pMsg->taskId = schGenUUID(); pMsg->taskType = TASK_TYPE_PERSISTENT; pMsg->contentLen = msgLen; - memcpy(pMsg->msg, msg, msgLen); + /*memcpy(pMsg->msg, ((SSubQueryMsg*)msg)->msg, msgLen);*/ tInfo.msg = pMsg; @@ -1512,9 +1513,7 @@ int32_t schedulerConvertDagToTaskList(SQueryDag* pDag, SArray **pTasks) { info = NULL; _return: - schedulerFreeTaskList(info); - SCH_RET(code); } diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index eb63f2b14479e1e4cb45a4e2b1bf015a7b4699a1..3cb5a655728cc77e477352bcd2425fce14c434df 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,5 +1,5 @@ -set(TDB_SUBDIRS "btree" "db" "hash" "mpool" "dmgr") +set(TDB_SUBDIRS "db") foreach(TDB_SUBDIR ${TDB_SUBDIRS}) aux_source_directory("src/${TDB_SUBDIR}" TDB_SRC) endforeach() diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index eee8f8ed33274a0834eea61b879e831c530d6021..faf9208f8f8f454cb94ba880fd14a30e17f47d24 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,30 +22,41 @@ extern "C" { #endif -#define TDB_EXTERN -#define TDB_PUBLIC -#define TDB_STATIC static - -typedef enum { - TDB_BTREE_T = 0, - TDB_HASH_T, - TDB_HEAP_T, -} tdb_db_t; - -// Forward declaration -typedef struct TDB TDB; -typedef struct TDB_CURSOR TDB_CURSOR; - -// SKey -typedef struct { - void* bdata; - uint32_t size; -} TDB_KEY, TDB_VALUE; - -// TDB Operations -TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type); -TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); -TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags); +// #define TDB_EXTERN +// #define TDB_PUBLIC +// #define TDB_STATIC static + +// typedef enum { TDB_BTREE_T = 0, TDB_HASH_T = 1, TDB_HEAP_T = 2 } tdb_db_t; + +// // Forward declarations +// typedef struct TDB TDB; +// // typedef struct TDB_MPOOL TDB_MPOOL; +// // typedef struct TDB_MPFILE TDB_MPFILE; +// // typedef struct TDB_CURSOR TDB_CURSOR; + +// typedef struct { +// void* bdata; +// uint32_t size; +// } TDB_KEY, TDB_VALUE; + +// // TDB Operations +// int tdbCreateDB(TDB** dbpp, tdb_db_t type); +// int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags); +// int tdbCloseDB(TDB* dbp, uint32_t flags); +// int tdbPut(TDB* dbp, const TDB_KEY* key, const TDB_VALUE* value, uint32_t flags); +// int tdbGet(TDB* dbp, const TDB_KEY* key, TDB_VALUE* value, uint32_t flags); + +// // TDB_MPOOL +// int tdbOpenMPool(TDB_MPOOL** mp); +// int tdbCloseMPool(TDB_MPOOL* mp); + +// // TDB_MPFILE +// int tdbOpenMPFile(TDB_MPFILE** mpf, TDB_MPOOL* mp); +// int tdbCloseMPFile(TDB_MPFILE** mpf); + +// // TDB_CURSOR +// int tdbOpenCursor(TDB* dbp, TDB_CURSOR** tdbcpp); +// int tdbCloseCurosr(TDB_CURSOR* tdbcp); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/db/tdbDB.c b/source/libs/tdb/src/db/tdbDB.c deleted file mode 100644 index eaf85ea4a1930e2a6304986df43656ee07cb8e20..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/db/tdbDB.c +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbDB.h" -#include "tdb.h" - -TDB_EXTERN int tdbCreateDB(TDB** dbpp, tdb_db_t type) { - TDB* dbp; - int ret; - - dbp = calloc(1, sizeof(*dbp)); - if (dbp == NULL) { - return -1; - } - - dbp->pageSize = TDB_DEFAULT_PGSIZE; - dbp->type = type; - - switch (type) { - case TDB_BTREE_T: - // ret = tdbInitBtreeDB(dbp); - // if (ret < 0) goto _err; - break; - case TDB_HASH_T: - // ret = tdbInitHashDB(dbp); - // if (ret < 0) goto _err; - break; - case TDB_HEAP_T: - // ret = tdbInitHeapDB(dbp); - // if (ret < 0) goto _err; - break; - default: - break; - } - - *dbpp = dbp; - return 0; - -_err: - if (dbp) { - free(dbp); - } - *dbpp = NULL; - return 0; -} - -TDB_EXTERN int tdbOpenDB(TDB* dbp, const char* fname, const char* dbname, uint32_t flags) { - int ret = 0; - - if ((dbp->fname = strdup(fname)) == NULL) { - ret = -1; - return ret; - } - - // Create the backup file if the file not exists - - // Open the file as a sub-db or a master-db - if (dbname) { - if ((dbp->dbname = strdup(dbname)) == NULL) { - ret = -1; - return ret; - } - // TODO: Open the DB as a SUB-DB in this file - } else { - // TODO: Open the DB as a MASTER-DB in this file - } - - return ret; -} - -TDB_EXTERN int tdbCloseDB(TDB* dbp, uint32_t flags) { - // TODO - return 0; -} \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbHeap.h b/source/libs/tdb/src/db/tdb_db.c similarity index 73% rename from source/libs/tdb/src/inc/tdbHeap.h rename to source/libs/tdb/src/db/tdb_db.c index 25a812fa5f7507624b6e4e02954735cbd0e38b00..3675844535319d0f35bd62b6f74a6c69c69bba82 100644 --- a/source/libs/tdb/src/inc/tdbHeap.h +++ b/source/libs/tdb/src/db/tdb_db.c @@ -13,23 +13,14 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_HEAP_H_ -#define _TD_TDB_HEAP_H_ +#include "tdb_db.h" -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { +int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags) { // TODO -} TDB_HEAP; - -TDB_PUBLIC int tdbInitHeapDB(TDB *dbp); - -#ifdef __cplusplus + return 0; } -#endif -#endif /*_TD_TDB_HEAP_H_*/ \ No newline at end of file +int tdbClose(TDB *dbp, uint32_t flags) { + // TODO + return 0; +} \ No newline at end of file diff --git a/source/libs/tdb/src/db/tdb_mpool.c b/source/libs/tdb/src/db/tdb_mpool.c new file mode 100644 index 0000000000000000000000000000000000000000..9b67f405a92781b709cd237335a85120cf8b0b78 --- /dev/null +++ b/source/libs/tdb/src/db/tdb_mpool.c @@ -0,0 +1,379 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdb_mpool.h" + +static int tdbGnrtFileID(const char *fname, uint8_t *fileid); +static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf); +static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid); +static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p); +static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p); +static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp); + +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize) { + TDB_MPOOL *mp = NULL; + size_t tsize; + pg_t * pagep; + + // check parameters + if (!TDB_IS_PGSIZE_VLD(pgsize)) { + tdbError("invalid page size"); + return -1; + } + + // allocate handle + mp = (TDB_MPOOL *)calloc(1, sizeof(*mp)); + if (mp == NULL) { + tdbError("failed to malloc memory pool handle"); + goto _err; + } + + // initialize the handle + mp->cachesize = cachesize; + mp->pgsize = pgsize; + mp->npages = cachesize / pgsize; + mp->clockHand = 0; + + TD_DLIST_INIT(&mp->freeList); + + mp->pages = (pg_t *)calloc(mp->npages, sizeof(pg_t)); + if (mp->pages == NULL) { + tdbError("failed to malloc memory pool pages"); + goto _err; + } + + for (frame_id_t i = 0; i < mp->npages; i++) { + mp->pages[i].p = malloc(pgsize); + if (mp->pages[i].p == NULL) { + goto _err; + } + + taosInitRWLatch(&mp->pages[i].rwLatch); + mp->pages[i].frameid = i; + mp->pages[i].pgid = TDB_IVLD_PGID; + + // add new page to the free list + TD_DLIST_APPEND_WITH_FIELD(&(mp->freeList), &(mp->pages[i]), free); + } + +#define PGTAB_FACTOR 1.0 + mp->pgtab.nbucket = mp->npages / PGTAB_FACTOR; + mp->pgtab.hashtab = (pg_list_t *)calloc(mp->pgtab.nbucket, sizeof(pg_list_t)); + if (mp->pgtab.hashtab == NULL) { + tdbError("failed to malloc memory pool hash table"); + goto _err; + } + + // return + *mpp = mp; + return 0; + +_err: + tdbMPoolClose(mp); + *mpp = NULL; + return -1; +} + +int tdbMPoolClose(TDB_MPOOL *mp) { + if (mp) { + tfree(mp->pgtab.hashtab); + if (mp->pages) { + for (int i = 0; i < mp->npages; i++) { + tfree(mp->pages[i].p); + } + + free(mp->pages); + } + + free(mp); + } + return 0; +} + +int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp) { + TDB_MPFILE *mpf; + + if ((mpf = (TDB_MPFILE *)calloc(1, sizeof(*mpf))) == NULL) { + return -1; + } + + mpf->fd = -1; + + if ((mpf->fname = strdup(fname)) == NULL) { + goto _err; + } + + if ((mpf->fd = open(fname, O_CREAT | O_RDWR, 0755)) < 0) { + goto _err; + } + + if (tdbGnrtFileID(fname, mpf->fileid) < 0) { + goto _err; + } + + // Register current MPF to MP + tdbMPoolRegFile(mp, mpf); + + *mpfp = mpf; + return 0; + +_err: + tdbMPoolFileClose(mpf); + *mpfp = NULL; + return -1; +} + +int tdbMPoolFileClose(TDB_MPFILE *mpf) { + if (mpf) { + if (mpf->fd > 0) { + close(mpf->fd); + } + tfree(mpf->fname); + free(mpf); + } + return 0; +} + +#define MPF_GET_PAGE_BUCKETID(fileid, pgno, nbuckets) \ + ({ \ + uint64_t *tmp = (uint64_t *)fileid; \ + (tmp[0] + tmp[1] + tmp[2] + (pgno)) % (nbuckets); \ + }) + +int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr) { + // TODO + return 0; +} + +int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr) { + // TODO + return 0; +} + +int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { + pg_t * pagep; + TDB_MPOOL *mp; + pg_list_t *pglist; + + mp = mpf->mp; + + // check if the page already in pool + pglist = mp->pgtab.hashtab + MPF_GET_PAGE_BUCKETID(mpf->fileid, pgno, mp->pgtab.nbucket); + pagep = TD_DLIST_HEAD(pglist); + while (pagep) { + if (memcmp(mpf->fileid, pagep->pgid.fileid, TDB_FILE_ID_LEN) == 0 && pgno == pagep->pgid.pgno) { + break; + } + + pagep = TD_DLIST_NODE_NEXT_WITH_FIELD(pagep, hash); + } + + if (pagep) { + // page is found + // todo: pin the page and return + *(void **)addr = pagep->p; + return 0; + } + + // page not found + pagep = TD_DLIST_HEAD(&mp->freeList); + if (pagep) { + // has free page + TD_DLIST_POP_WITH_FIELD(&(mp->freeList), pagep, free); + } else { + // no free page available + tdbMPoolClockEvictPage(mp, &pagep); + if (pagep) { + if (pagep->dirty) { + // TODO: Handle dirty page eviction + } + } + } + + if (pagep == NULL) { + // no available container page + return -1; + } + + // load page from the disk if a container page is available + // TODO: load the page from the disk + if (tdbMPoolFileReadPage(mpf, pgno, pagep->p) < 0) { + return -1; + } + + memcpy(pagep->pgid.fileid, mpf->fileid, TDB_FILE_ID_LEN); + pagep->pgid.pgno = pgno; + pagep->dirty = 0; + pagep->pinRef = 1; + + // add current page to page table + TD_DLIST_APPEND_WITH_FIELD(pglist, pagep, hash); + + return 0; +} + +int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr) { + // TODO + return 0; +} + +static int tdbGnrtFileID(const char *fname, uint8_t *fileid) { + struct stat statbuf; + + if (stat(fname, &statbuf) < 0) { + return -1; + } + + memset(fileid, 0, TDB_FILE_ID_LEN); + + ((uint64_t *)fileid)[0] = (uint64_t)statbuf.st_ino; + ((uint64_t *)fileid)[1] = (uint64_t)statbuf.st_dev; + ((uint64_t *)fileid)[2] = rand(); + + return 0; +} + +#define MPF_GET_BUCKETID(fileid) \ + ({ \ + uint64_t *tmp = (uint64_t *)fileid; \ + (tmp[0] + tmp[1] + tmp[2]) % MPF_HASH_BUCKETS; \ + }) + +static void tdbMPoolRegFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + mpf_bucket_t *bktp; + + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); + + taosWLockLatch(&(bktp->latch)); + + TD_DLIST_APPEND_WITH_FIELD(bktp, mpf, node); + + taosWUnLockLatch(&(bktp->latch)); + + mpf->mp = mp; +} + +static TDB_MPFILE *tdbMPoolGetFile(TDB_MPOOL *mp, uint8_t *fileid) { + TDB_MPFILE * mpf = NULL; + mpf_bucket_t *bktp; + + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(fileid); + + taosRLockLatch(&(bktp->latch)); + + mpf = TD_DLIST_HEAD(bktp); + while (mpf) { + if (memcmp(fileid, mpf->fileid, TDB_FILE_ID_LEN) == 0) { + break; + } + + mpf = TD_DLIST_NODE_NEXT_WITH_FIELD(mpf, node); + } + + taosRUnLockLatch(&(bktp->latch)); + + return mpf; +} + +static void tdbMPoolUnregFile(TDB_MPOOL *mp, TDB_MPFILE *mpf) { + mpf_bucket_t *bktp; + TDB_MPFILE * tmpf; + + if (mpf->mp == NULL) return; + + ASSERT(mpf->mp == mp); + + bktp = mp->mpfht.buckets + MPF_GET_BUCKETID(mpf->fileid); + + taosWLockLatch(&(bktp->latch)); + + tmpf = TD_DLIST_HEAD(bktp); + + while (tmpf) { + if (memcmp(mpf->fileid, tmpf->fileid, TDB_FILE_ID_LEN) == 0) { + TD_DLIST_POP_WITH_FIELD(bktp, tmpf, node); + break; + } + + tmpf = TD_DLIST_NODE_NEXT_WITH_FIELD(tmpf, node); + } + + taosWUnLockLatch(&(bktp->latch)); + + ASSERT(tmpf == mpf); +} + +static int tdbMPoolFileReadPage(TDB_MPFILE *mpf, pgno_t pgno, void *p) { + pgsize_t pgsize; + TDB_MPOOL *mp; + off_t offset; + size_t rsize; + + mp = mpf->mp; + pgsize = mp->pgsize; + offset = pgno * pgsize; + + // TODO: use loop to read all data + rsize = pread(mpf->fd, p, pgsize, offset); + // TODO: error handle + + return 0; +} + +static int tdbMPoolFileWritePage(TDB_MPFILE *mpf, pgno_t pgno, const void *p) { + pgsize_t pgsize; + TDB_MPOOL *mp; + off_t offset; + + mp = mpf->mp; + pgsize = mp->pgsize; + offset = pgno * pgsize; + + lseek(mpf->fd, offset, SEEK_SET); + // TODO: handle error + + write(mpf->fd, p, pgsize); + // TODO: handle error + + return 0; +} + +static void tdbMPoolClockEvictPage(TDB_MPOOL *mp, pg_t **pagepp) { + pg_t * pagep; + frame_id_t och; + + *pagepp = NULL; + och = mp->clockHand; + + do { + pagep = mp->pages + mp->clockHand; + mp->clockHand = (mp->clockHand + 1) % mp->npages; + + if (pagep->pinRef == 0) { + if (pagep->rbit == 1) { + pagep->rbit = 0; + } else { + break; + } + } + + if (mp->clockHand == och) { + return; + } + } while (1); + + *pagepp = pagep; +} \ No newline at end of file diff --git a/source/libs/tdb/src/dmgr/tdbDiskMgr.c b/source/libs/tdb/src/dmgr/tdbDiskMgr.c deleted file mode 100644 index 71ab5f2589d7c56bbfe3e0a493f10fa54d085463..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/dmgr/tdbDiskMgr.c +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tdbDiskMgr.h" - -struct STkvDiskMgr { - char * fname; - uint16_t pgsize; - FileFd fd; - pgid_t npgid; -}; - -#define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) - -int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { - STkvDiskMgr *pDiskMgr; - - pDiskMgr = malloc(sizeof(*pDiskMgr)); - if (pDiskMgr == NULL) { - return -1; - } - - pDiskMgr->fname = strdup(fname); - if (pDiskMgr->fname == NULL) { - free(pDiskMgr); - return -1; - } - pDiskMgr->pgsize = pgsize; - pDiskMgr->fd = open(fname, O_CREAT | O_RDWR, 0755); - if (pDiskMgr->fd < 0) { - free(pDiskMgr->fname); - free(pDiskMgr); - return -1; - } - - *ppDiskMgr = pDiskMgr; - - return 0; -} - -int tdmClose(STkvDiskMgr *pDiskMgr) { - close(pDiskMgr->fd); - free(pDiskMgr->fname); - free(pDiskMgr); - return 0; -} - -int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { - taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); - taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); - return 0; -} - -int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { - taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); - taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); - return 0; -} - -int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } - -int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h deleted file mode 100644 index 28258b8e60b7ea44f13fc56903d99e7427cdddfc..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_BTREE_H_ -#define _TD_TDB_BTREE_H_ - -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - pgid_t root; // root page number -} TDB_BTREE; - -TDB_PUBLIC int tdbInitBtreeDB(TDB *dbp); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBufPool.h b/source/libs/tdb/src/inc/tdbBufPool.h deleted file mode 100644 index 5200d22faaa8153d2e71265b0fbc22b7158af3d6..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbBufPool.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_BUF_POOL_H_ -#define _TD_TDB_BUF_POOL_H_ - -#include "tdbPage.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct STdbBufPool STdbBufPool; - -int tbpOpen(STdbBufPool **ppTkvBufPool); -int tbpClose(STdbBufPool *pTkvBufPool); -STdbPage *tbpNewPage(STdbBufPool *pTkvBufPool); -int tbpDelPage(STdbBufPool *pTkvBufPool); -STdbPage *tbpFetchPage(STdbBufPool *pTkvBufPool, pgid_t pgid); -int tbpUnpinPage(STdbBufPool *pTkvBufPool, pgid_t pgid); -void tbpFlushPages(STdbBufPool *pTkvBufPool); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_BUF_POOL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDiskMgr.h b/source/libs/tdb/src/inc/tdbDiskMgr.h deleted file mode 100644 index b83a1474371a6cbd3f0463ae5a2f160ae499eac5..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbDiskMgr.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDISK_MGR_H_ -#define _TD_TDISK_MGR_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" - -#include "tdbDef.h" - -typedef struct STkvDiskMgr STkvDiskMgr; - -int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); -int tdmClose(STkvDiskMgr *pDiskMgr); -int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData); -int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); -int tdmFlush(STkvDiskMgr *pDiskMgr); -pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDISK_MGR_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbHash.h b/source/libs/tdb/src/inc/tdbHash.h deleted file mode 100644 index 8219bda2f82eae05f7aa1fba5433c261272612b5..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbHash.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_HASH_H_ -#define _TD_TDB_HASH_H_ - -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - // TODO -} TDB_HASH; - -TDB_PUBLIC int tdbInitHashDB(TDB *dbp); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_HASH_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbPage.h b/source/libs/tdb/src/inc/tdbPage.h deleted file mode 100644 index e7245b6c393be112a0b5ab9c64085aff5ae42615..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/inc/tdbPage.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TKV_PAGE_H_ -#define _TD_TKV_PAGE_H_ - -#include "os.h" -#include "tdbDef.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - pgid_t pgid; - int32_t pinCount; - bool idDirty; - char* pData; -} STdbPage; - -typedef struct { - uint16_t dbver; - uint16_t pgsize; - uint32_t cksm; -} STdbPgHdr; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TKV_PAGE_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdb_db.h similarity index 62% rename from source/libs/tdb/src/inc/tdbDB.h rename to source/libs/tdb/src/inc/tdb_db.h index d0ef9e22d0f625889ee2403c2820b6c3a5fc54d2..5f0529462b0f592b3d3adf927865c048d882b6f3 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdb_db.h @@ -16,38 +16,28 @@ #ifndef _TD_TDB_DB_H_ #define _TD_TDB_DB_H_ -#include "tdb.h" -#include "tdbBtree.h" -#include "tdbHash.h" -#include "tdbHeap.h" +#include "tdb_mpool.h" #ifdef __cplusplus extern "C" { #endif -typedef struct { - // TODO -} TDB_MPOOL; - -typedef struct { - int fd; -} TDB_FH; +typedef struct TDB TDB; struct TDB { - pgsize_t pageSize; - tdb_db_t type; - char * fname; - char * dbname; - union { - TDB_BTREE *btree; - TDB_HASH * hash; - TDB_HEAP * heap; - } dbam; // db access method - - TDB_FH * fhp; // The backup file handle - TDB_MPOOL *mph; // The memory pool handle + char * fname; + char * dbname; + TDB_MPFILE *mpf; + // union { + // TDB_BTREE *btree; + // TDB_HASH * hash; + // TDB_HEAP * heap; + // } dbam; // db access method }; +int tdbOpen(TDB **dbpp, const char *fname, const char *dbname, uint32_t flags); +int tdbClose(TDB *dbp, uint32_t flags); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbDef.h b/source/libs/tdb/src/inc/tdb_inc.h similarity index 65% rename from source/libs/tdb/src/inc/tdbDef.h rename to source/libs/tdb/src/inc/tdb_inc.h index 4b5e54368bddd630c21db87a80dd398c8354fce1..885191477cfd3e760959a4f81d1dce7f5ba4475e 100644 --- a/source/libs/tdb/src/inc/tdbDef.h +++ b/source/libs/tdb/src/inc/tdb_inc.h @@ -13,18 +13,30 @@ * along with this program. If not, see . */ -#ifndef _TD_TDB_DEF_H_ -#define _TD_TDB_DEF_H_ +#ifndef _TD_TDB_INC_H_ +#define _TD_TDB_INC_H_ #include "os.h" +#include "tlist.h" +#include "tlockfree.h" #ifdef __cplusplus extern "C" { #endif +// pgno_t +typedef int32_t pgno_t; +#define TDB_IVLD_PGNO ((pgno_t)-1) + +// fileid +#define TDB_FILE_ID_LEN 24 + // pgid_t -typedef int32_t pgid_t; -#define TDB_IVLD_PGID ((pgid_t)-1) +typedef struct { + uint8_t fileid[TDB_FILE_ID_LEN]; + pgno_t pgno; +} pgid_t; +#define TDB_IVLD_PGID (pgid_t){0, TDB_IVLD_PGNO}; // framd_id_t typedef int32_t frame_id_t; @@ -34,10 +46,13 @@ typedef int32_t pgsize_t; #define TDB_MIN_PGSIZE 512 #define TDB_MAX_PGSIZE 16384 #define TDB_DEFAULT_PGSIZE 4096 -#define TDB_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE)) +#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE)) + +// tdb_log +#define tdbError(var) #ifdef __cplusplus } #endif -#endif /*_TD_TDB_DEF_H_*/ \ No newline at end of file +#endif /*_TD_TDB_INC_H_*/ diff --git a/source/libs/tdb/src/inc/tdb_mpool.h b/source/libs/tdb/src/inc/tdb_mpool.h new file mode 100644 index 0000000000000000000000000000000000000000..37c82f38335b4bda0ce8835f5b47f9178a60c5d1 --- /dev/null +++ b/source/libs/tdb/src/inc/tdb_mpool.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TDB_MPOOL_H_ +#define _TD_TDB_MPOOL_H_ + +#include "tdb_inc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Exposed handle +typedef struct TDB_MPOOL TDB_MPOOL; +typedef struct TDB_MPFILE TDB_MPFILE; + +typedef TD_DLIST_NODE(pg_t) pg_free_dlist_node_t, pg_hash_dlist_node_t; +typedef struct pg_t { + SRWLatch rwLatch; + frame_id_t frameid; + pgid_t pgid; + uint8_t dirty; + uint8_t rbit; + int32_t pinRef; + pg_free_dlist_node_t free; + pg_hash_dlist_node_t hash; + void * p; +} pg_t; + +typedef TD_DLIST(pg_t) pg_list_t; +typedef struct { + SRWLatch latch; + TD_DLIST(TDB_MPFILE); +} mpf_bucket_t; +struct TDB_MPOOL { + int64_t cachesize; + pgsize_t pgsize; + int32_t npages; + pg_t * pages; + pg_list_t freeList; + frame_id_t clockHand; + struct { + int32_t nbucket; + pg_list_t *hashtab; + } pgtab; // page table, hash + struct { +#define MPF_HASH_BUCKETS 16 + mpf_bucket_t buckets[MPF_HASH_BUCKETS]; + } mpfht; // MPF hash table. MPFs using this MP will be put in this hash table +}; + +#define MP_PAGE_AT(mp, idx) (mp)->pages[idx] + +typedef TD_DLIST_NODE(TDB_MPFILE) td_mpf_dlist_node_t; +struct TDB_MPFILE { + char * fname; // file name + int fd; // fd + uint8_t fileid[TDB_FILE_ID_LEN]; // file ID + TDB_MPOOL * mp; // underlying memory pool + td_mpf_dlist_node_t node; +}; + +/*=================================================== Exposed apis ==================================================*/ +// TDB_MPOOL +int tdbMPoolOpen(TDB_MPOOL **mpp, uint64_t cachesize, pgsize_t pgsize); +int tdbMPoolClose(TDB_MPOOL *mp); +int tdbMPoolSync(TDB_MPOOL *mp); + +// TDB_MPFILE +int tdbMPoolFileOpen(TDB_MPFILE **mpfp, const char *fname, TDB_MPOOL *mp); +int tdbMPoolFileClose(TDB_MPFILE *mpf); +int tdbMPoolFileNewPage(TDB_MPFILE *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileFreePage(TDB_MPOOL *mpf, pgno_t *pgno, void *addr); +int tdbMPoolFileGetPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFilePutPage(TDB_MPFILE *mpf, pgno_t pgno, void *addr); +int tdbMPoolFileSync(TDB_MPFILE *mpf); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TDB_MPOOL_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/src/mpool/tdbBufPool.c b/source/libs/tdb/src/mpool/tdbBufPool.c deleted file mode 100644 index bc3c386b0fc23a5796decefa4d959a53d6a35cd6..0000000000000000000000000000000000000000 --- a/source/libs/tdb/src/mpool/tdbBufPool.c +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "thash.h" -#include "tlist.h" - -#include "tdbBufPool.h" -#include "tdbDiskMgr.h" -#include "tdbPage.h" - -struct SFrameIdWrapper { - TD_SLIST_NODE(SFrameIdWrapper); - frame_id_t id; -}; - -struct STdbBufPool { - STdbPage* pages; - STkvDiskMgr* pDiskMgr; - SHashObj* pgTb; // page_id_t --> frame_id_t - TD_SLIST(SFrameIdWrapper) freeList; - pthread_mutex_t mutex; -}; - -typedef struct STkvLRUReplacer { -} STkvLRUReplacer; - -typedef struct STkvLFUReplacer { -} STkvLFUReplacer; - -typedef struct STkvCLKReplacer { -} STkvCLKReplacer; - -typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t; - -typedef struct STkvReplacer { - tkv_replacer_t type; - union { - STkvLRUReplacer lruRep; - STkvLFUReplacer lfuRep; - STkvCLKReplacer clkRep; - }; -} STkvReplacer; \ No newline at end of file diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index 2d77c1f4e9124bd07d8ecb3e27bf9db9103f33e7..7fbfaf550691c5e12409a3c8540ab7fbfc080a4f 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -1,3 +1,7 @@ +# tdbMPoolTest +add_executable(tdbMPoolTest "tdbMPoolTest.cpp") +target_link_libraries(tdbMPoolTest tdb gtest gtest_main) + # tdbTest add_executable(tdbTest "tdbTest.cpp") target_link_libraries(tdbTest tdb gtest gtest_main) \ No newline at end of file diff --git a/source/libs/tdb/test/tdbMPoolTest.cpp b/source/libs/tdb/test/tdbMPoolTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..17381759fbc9dc7b9f8f5a79810eb22bf016c056 --- /dev/null +++ b/source/libs/tdb/test/tdbMPoolTest.cpp @@ -0,0 +1,31 @@ +#include "gtest/gtest.h" + +#include + +#include "tdb_mpool.h" + +TEST(tdb_mpool_test, test1) { + TDB_MPOOL * mp; + TDB_MPFILE *mpf; + pgno_t pgno; + void * pgdata; + + // open mp + tdbMPoolOpen(&mp, 16384, 4096); + + // open mpf + tdbMPoolFileOpen(&mpf, "test.db", mp); + +#define TEST1_TOTAL_PAGES 100 + for (int i = 0; i < TEST1_TOTAL_PAGES; i++) { + tdbMPoolFileNewPage(mpf, &pgno, pgdata); + + *(pgno_t *)pgdata = i; + } + + // close mpf + tdbMPoolFileClose(mpf); + + // close mp + tdbMPoolClose(mp); +} diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index f27e17f1ca8f45c8c15feb423d6b706dece79f01..113bb2560f5ad8a7d71ac253f7514eec39a2a7a8 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -3,8 +3,8 @@ #include "tdb.h" TEST(tdb_api_test, tdb_create_open_close_db_test) { - int ret; - TDB *dbp; + // int ret; + // TDB *dbp; // tdbCreateDB(&dbp, TDB_BTREE_T); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index c760acd52e80443d2a4bb7b5874ce7258e687798..c7a23a2482baf09d3c0f1257b5317b6098da330c 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -123,9 +123,9 @@ typedef struct { } SRpcReqContext; typedef struct { - SRpcInfo* pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void* ahandle; // handle provided by app + SRpcInfo* pTransInst; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void* ahandle; // handle provided by app // struct SRpcConn* pConn; // pConn allocated tmsg_t msgType; // message type uint8_t* pCont; // content provided by app @@ -182,7 +182,7 @@ typedef struct { #define TRANS_RESERVE_SIZE (sizeof(STranConnCtx)) -#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead) + sizeof(STransDigestMsg)) +#define TRANS_MSG_OVERHEAD (sizeof(STransMsgHead)) #define transHeadFromCont(cont) ((STransMsgHead*)((char*)cont - sizeof(STransMsgHead))) #define transContFromHead(msg) (msg + sizeof(STransMsgHead)) #define transMsgLenFromCont(contLen) (contLen + sizeof(STransMsgHead)) @@ -201,6 +201,7 @@ bool transDecompressMsg(char* msg, int32_t len, int32_t* flen); void transConnCtxDestroy(STransConnCtx* ctx); +void transFreeMsg(void* msg); typedef struct SConnBuffer { char* buf; int len; diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index d870ae98ab886f17ea866f27dcde381fc73c6242..b0927e9ecfbb61da5508fb4e1151621e600897e4 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -42,6 +42,8 @@ int tsRpcMaxRetry; int tsRpcHeadSize; int tsRpcOverhead; +SHashObj *tsFqdnHash; + #ifndef USE_UV typedef struct { @@ -215,6 +217,8 @@ static void rpcInitImp(void) { tsRpcOverhead = sizeof(SRpcReqContext); tsRpcRefId = taosOpenRef(200, rpcFree); + + tsFqdnHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); } int32_t rpcInit(void) { @@ -224,6 +228,9 @@ int32_t rpcInit(void) { void rpcCleanup(void) { taosCloseRef(tsRpcRefId); + taosHashClear(tsFqdnHash); + taosHashCleanup(tsFqdnHash); + tsFqdnHash = NULL; tsRpcRefId = -1; } @@ -235,11 +242,14 @@ void *rpcOpen(const SRpcInit *pInit) { pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; - if (pInit->label) tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); + if (pInit->label) tstrncpy(pRpc->label, pInit->label, tListLen(pInit->label)); pRpc->connType = pInit->connType; if (pRpc->connType == TAOS_CONN_CLIENT) { pRpc->numOfThreads = pInit->numOfThreads; + if (pRpc->numOfThreads >= 10) { + pRpc->numOfThreads = 10; + } } else { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } @@ -571,7 +581,17 @@ static void rpcFreeMsg(void *msg) { static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) { SRpcConn *pConn; - uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn); + uint32_t peerIp = 0; + uint32_t *pPeerIp = taosHashGet(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1); + if (pPeerIp != NULL) { + peerIp = *pPeerIp; + } else { + peerIp = taosGetIpv4FromFqdn(peerFqdn); + if (peerIp != 0xFFFFFFFF) { + taosHashPut(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1, &peerIp, sizeof(peerIp)); + } + } + if (peerIp == 0xFFFFFFFF) { tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); terrno = TSDB_CODE_RPC_FQDN_ERROR; @@ -752,8 +772,8 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { } taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); - tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s spi:%d", pRpc->label, pConn, pConn->linkUid, sid, - hashstr, pConn->spi); + tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s spi:%d", pRpc->label, pConn, pConn->linkUid, + sid, hashstr, pConn->spi); } return pConn; @@ -1340,9 +1360,14 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType), pConn->peerFqdn, pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } else { - if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured - tDebug("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType), - pConn->peerIp, pConn->peerPort, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + if (pHead->code == 0) { + pConn->secured = 1; // for success response, set link as secured + } + + char ipport[40] = {0}; + taosIpPort2String(pConn->peerIp, pConn->peerPort, ipport); + tDebug("%s, %s is sent to %s, code:0x%x len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType), + ipport, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } // tTrace("connection type is: %d", pConn->connType); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index bfadfe56eff19e4738d53a9f1209bac421d16095..f265acf8c1fd15b18a5be1625166c423ea34f3fd 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -30,6 +30,7 @@ typedef struct SCliConn { char spi; char secured; uint64_t expireTime; + int8_t notifyCount; // timers already notify to client } SCliConn; typedef struct SCliMsg { @@ -72,8 +73,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* co // register timer in each thread to clear expire conn static void clientTimeoutCb(uv_timer_t* handle); -// process data read from server, auth/decompress etc later -static void clientHandleResp(SCliConn* conn); // check whether already read complete packet from server static bool clientReadComplete(SConnBuffer* pBuf); // alloc buf for read @@ -88,10 +87,15 @@ static void clientAsyncCb(uv_async_t* handle); static void clientDestroy(uv_handle_t* handle); static void clientConnDestroy(SCliConn* pConn, bool clear /*clear tcp handle or not*/); -static void clientMsgDestroy(SCliMsg* pMsg); +// process data read from server, auth/decompress etc later +static void clientHandleResp(SCliConn* conn); +// handle except about conn +static void clientHandleExcept(SCliConn* conn); // handle req from app static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); +static void clientMsgDestroy(SCliMsg* pMsg); +static void destroyTransConnCtx(STransConnCtx* ctx); // thread obj static SCliThrdObj* createThrdObj(); static void destroyThrdObj(SCliThrdObj* pThrd); @@ -100,22 +104,50 @@ static void* clientThread(void* arg); static void clientHandleResp(SCliConn* conn) { STransConnCtx* pCtx = ((SCliMsg*)conn->data)->ctx; - SRpcInfo* pRpc = pCtx->pRpc; - SRpcMsg rpcMsg; + SRpcInfo* pRpc = pCtx->pTransInst; + + STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); + pHead->code = htonl(pHead->code); + pHead->msgLen = htonl(pHead->msgLen); - rpcMsg.pCont = conn->readBuf.buf; - rpcMsg.contLen = conn->readBuf.len; + SRpcMsg rpcMsg; + rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); + rpcMsg.pCont = transContFromHead(pHead); + rpcMsg.code = pHead->code; + rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; + (pRpc->cfp)(NULL, &rpcMsg, NULL); + conn->notifyCount += 1; SCliThrdObj* pThrd = conn->hostThrd; + tfree(conn->data); addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); + + // start thread's timer of conn pool if not active if (!uv_is_active((uv_handle_t*)pThrd->pTimer) && pRpc->idleTime > 0) { uv_timer_start((uv_timer_t*)pThrd->pTimer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); } - free(pCtx->ip); - free(pCtx); - // impl + destroyTransConnCtx(pCtx); +} +static void clientHandleExcept(SCliConn* pConn) { + SCliMsg* pMsg = pConn->data; + + STransConnCtx* pCtx = pMsg->ctx; + SRpcInfo* pRpc = pCtx->pTransInst; + + transFreeMsg((pMsg->msg.pCont)); + pMsg->msg.pCont = NULL; + + SRpcMsg rpcMsg = {0}; + rpcMsg.ahandle = pCtx->ahandle; + rpcMsg.code = -1; + // SRpcInfo* pRpc = pMsg->ctx->pRpc; + (pRpc->cfp)(NULL, &rpcMsg, NULL); + tfree(pConn->data); + pConn->notifyCount += 1; + destroyTransConnCtx(pCtx); + clientConnDestroy(pConn, true); } static void clientTimeoutCb(uv_timer_t* handle) { @@ -191,6 +223,7 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { SRpcInfo* pRpc = ((SCliThrdObj*)conn->hostThrd)->pTransInst; conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); + conn->notifyCount = 0; // list already create before assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); @@ -246,19 +279,21 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf if (nread > 0) { pBuf->len += nread; if (clientReadComplete(pBuf)) { - tDebug("alread read complete"); + tDebug("conn %p read complete", conn); clientHandleResp(conn); } else { - tDebug("read half packet, continue to read"); + tDebug("conn %p read partial packet, continue to read", conn); } return; } assert(nread <= 0); if (nread == 0) { + tError("conn %p closed", conn); return; } - if (nread != UV_EOF) { - tDebug("read error %s", uv_err_name(nread)); + if (nread < 0) { + tError("conn %p read error: %s", conn, uv_err_name(nread)); + clientHandleExcept(conn); } // tDebug("Read error %s\n", uv_err_name(nread)); // uv_close((uv_handle_t*)handle, clientDestroy); @@ -282,19 +317,24 @@ static void clientDestroy(uv_handle_t* handle) { static void clientWriteCb(uv_write_t* req, int status) { SCliConn* pConn = req->data; + + SCliMsg* pMsg = pConn->data; + transFreeMsg((pMsg->msg.pCont)); + pMsg->msg.pCont = NULL; + if (status == 0) { - tDebug("data already was written on stream"); + tDebug("conn %p data already was written out", pConn); } else { - tError("failed to write: %s", uv_err_name(status)); - clientConnDestroy(pConn, true); + tError("conn %p failed to write: %s", pConn, uv_err_name(status)); + clientHandleExcept(pConn); return; } SCliThrdObj* pThrd = pConn->hostThrd; - if (pConn->stream == NULL) { - pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); - uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream); - pConn->stream->data = pConn; - } + // if (pConn->stream == NULL) { + // pConn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); + // uv_tcp_init(pThrd->loop, (uv_tcp_t*)pConn->stream); + // pConn->stream->data = pConn; + //} uv_read_start((uv_stream_t*)pConn->stream, clientAllocReadBufferCb, clientReadCb); // impl later } @@ -310,30 +350,19 @@ static void clientWrite(SCliConn* pConn) { pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - tDebug("data write out, msgType : %d, len: %d", pHead->msgType, msgLen); + tDebug("conn %p data write out, msgType : %d, len: %d", pConn, pHead->msgType, msgLen); uv_write(pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, clientWriteCb); } static void clientConnCb(uv_connect_t* req, int status) { // impl later SCliConn* pConn = req->data; - SCliMsg* pMsg = pConn->data; - - STransConnCtx* pCtx = pMsg->ctx; - SRpcInfo* pRpc = pCtx->pRpc; - if (status != 0) { // tError("failed to connect server(%s, %d), errmsg: %s", pCtx->ip, pCtx->port, uv_strerror(status)); - tError("failed to connect server, errmsg: %s", uv_strerror(status)); - // call user fp later - SRpcMsg rpcMsg; - rpcMsg.ahandle = pCtx->ahandle; - // SRpcInfo* pRpc = pMsg->ctx->pRpc; - (pRpc->cfp)(NULL, &rpcMsg, NULL); - - clientConnDestroy(pConn, true); - // uv_close((uv_handle_t*)req->handle, clientDestroy); + tError("conn %p failed to connect server: %s", pConn, uv_strerror(status)); + clientHandleExcept(pConn); return; } + tDebug("conn %p create", pConn); assert(pConn->stream == req->handle); clientWrite(pConn); @@ -349,6 +378,7 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); if (conn != NULL) { // impl later + tDebug("conn %p get from conn pool", conn); conn->data = pMsg; conn->writeReq->data = conn; @@ -462,6 +492,13 @@ static void destroyThrdObj(SCliThrdObj* pThrd) { free(pThrd->loop); free(pThrd); } + +static void destroyTransConnCtx(STransConnCtx* ctx) { + if (ctx != NULL) { + free(ctx->ip); + } + free(ctx); +} // void taosCloseClient(void* arg) { // impl later @@ -472,7 +509,6 @@ void taosCloseClient(void* arg) { free(cli->pThreadObj); free(cli); } - void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { // impl later char* ip = (char*)(pEpSet->fqdn[pEpSet->inUse]); @@ -487,7 +523,7 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); - pCtx->pRpc = (SRpcInfo*)shandle; + pCtx->pTransInst = (SRpcInfo*)shandle; pCtx->ahandle = pMsg->ahandle; pCtx->msgType = pMsg->msgType; pCtx->ip = strdup(ip); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 617abeea3953b120a876763020e1e7ffcb035307..5bece11becbd0f1018c370c499bd330aac0930ee 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -191,4 +191,11 @@ void transConnCtxDestroy(STransConnCtx* ctx) { free(ctx->ip); free(ctx); } + +void transFreeMsg(void* msg) { + if (msg == NULL) { + return; + } + free((char*)msg - sizeof(STransMsgHead)); +} #endif diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index b519a35f2491f4041263de8a03aaa1d7f976a71b..c70b1a5b28b97f7bfa01ee2e14fae5a767f6816c 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -16,6 +16,7 @@ #ifdef USE_UV #include "transComm.h" + typedef struct SConn { uv_tcp_t* pTcp; uv_write_t* pWriter; @@ -26,7 +27,6 @@ typedef struct SConn { int ref; int persist; // persist connection or not SConnBuffer connBuf; // read buf, - int count; int inType; void* pTransInst; // rpc init void* ahandle; // @@ -226,7 +226,7 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) { tDebug("%p timeout since no activity", conn); } -static void uvProcessData(SConn* pConn) { +static void uvHandleReq(SConn* pConn) { SRecvInfo info; SRecvInfo* p = &info; SConnBuffer* pBuf = &pConn->connBuf; @@ -271,6 +271,7 @@ static void uvProcessData(SConn* pConn) { rpcMsg.ahandle = NULL; rpcMsg.handle = pConn; + pConn->ref++; (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // auth @@ -283,20 +284,23 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { SConnBuffer* pBuf = &conn->connBuf; if (nread > 0) { pBuf->len += nread; - tDebug("on read %p, total read: %d, current read: %d", cli, pBuf->len, (int)nread); + tDebug("conn %p read summroy, total read: %d, current read: %d", conn, pBuf->len, (int)nread); if (readComplete(pBuf)) { - tDebug("alread read complete packet"); - uvProcessData(conn); + tDebug("conn %p alread read complete packet", conn); + uvHandleReq(conn); } else { - tDebug("read half packet, continue to read"); + tDebug("conn %p read partial packet, continue to read", conn); } return; } if (nread == 0) { + tDebug("conn %p except read", conn); + // destroyConn(conn, true); return; } if (nread != UV_EOF) { - tDebug("read error %s", uv_err_name(nread)); + tDebug("conn %p read error: %s", conn, uv_err_name(nread)); + destroyConn(conn, true); } } void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { @@ -306,7 +310,8 @@ void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* b void uvOnTimeoutCb(uv_timer_t* handle) { // opt - tDebug("time out"); + SConn* pConn = handle->data; + tDebug("conn %p time out", pConn); } void uvOnWriteCb(uv_write_t* req, int status) { @@ -316,10 +321,14 @@ void uvOnWriteCb(uv_write_t* req, int status) { buf->len = 0; memset(buf->buf, 0, buf->cap); buf->left = -1; + + SRpcMsg* pMsg = &conn->sendMsg; + transFreeMsg(pMsg->pCont); + if (status == 0) { - tDebug("data already was written on stream"); + tDebug("conn %p data already was written on stream", conn); } else { - tDebug("failed to write data, %s", uv_err_name(status)); + tDebug("conn %p failed to write data, %s", conn, uv_err_name(status)); destroyConn(conn, true); } // opt @@ -334,7 +343,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { static void uvPrepareSendData(SConn* conn, uv_buf_t* wb) { // impl later; - tDebug("prepare to send back"); + tDebug("conn %p prepare to send resp", conn); SRpcMsg* pMsg = &conn->sendMsg; if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); @@ -427,6 +436,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { assert(pending == UV_TCP); SConn* pConn = createConn(); + pConn->pTransInst = pThrd->pTransInst; /* init conn timer*/ pConn->pTimer = malloc(sizeof(uv_timer_t)); @@ -448,7 +458,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { if (uv_accept(q, (uv_stream_t*)(pConn->pTcp)) == 0) { uv_os_fd_t fd; uv_fileno((const uv_handle_t*)pConn->pTcp, &fd); - tDebug("new connection created: %d", fd); + tDebug("conn %p created, fd: %d", pConn, fd); uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocReadBufferCb, uvOnReadCb); } else { tDebug("failed to create new connection"); @@ -515,19 +525,19 @@ void* workerThread(void* arg) { static SConn* createConn() { SConn* pConn = (SConn*)calloc(1, sizeof(SConn)); + ++pConn->ref; return pConn; } -static void connCloseCb(uv_handle_t* handle) { - // impl later - // -} + static void destroyConn(SConn* conn, bool clear) { if (conn == NULL) { return; } + if (--conn->ref == 0) { + return; + } if (clear) { - uv_handle_t handle = *((uv_handle_t*)conn->pTcp); - uv_close(&handle, NULL); + uv_close((uv_handle_t*)conn->pTcp, NULL); } uv_timer_stop(conn->pTimer); free(conn->pTimer); @@ -646,6 +656,7 @@ void rpcSendResponse(const SRpcMsg* pMsg) { pthread_mutex_lock(&pThrd->connMtx); QUEUE_PUSH(&pThrd->conn, &pConn->queue); pthread_mutex_unlock(&pThrd->connMtx); + tDebug("conn %p start to send resp", pConn); uv_async_send(pConn->pWorkerAsync); } diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index fd3496cc17962e6f1ec9567d3088c1477db799ea..84814f39fc07d9a22635cd534748c433f2458b7a 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -63,7 +63,7 @@ static void *sendRequest(void *param) { if (pInfo->num % 20000 == 0) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); // tsem_wait(&pInfo->rspSem); tsem_wait(&pInfo->rspSem); - tDebug("recv response"); + tDebug("recv response succefully"); // usleep(100000000); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3a67b6515bf62ddb8118ba6bfbae88cbd529790c..56919ff99e988dfef86dabd0280bf8e1d2eb5ef1 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -248,6 +248,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg") // mnode-trans TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_ALREADY_EXIST, "Transaction already exists") diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 5cb149d53c20b94a4dbbe6d4ed7f9adb9082029f..8125f550d05df47d6f1b9880ee82c82e83a307c0 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -13,35 +13,36 @@ * along with this program. If not, see . */ -#include "os.h" - -#include "taoserror.h" +#define _DEFAULT_SOURCE #include "tqueue.h" +#include "taoserror.h" #include "ulog.h" typedef struct STaosQnode STaosQnode; typedef struct STaosQnode { STaosQnode *next; + STaosQueue *queue; char item[]; } STaosQnode; typedef struct STaosQueue { int32_t itemSize; int32_t numOfItems; - STaosQnode *head; - STaosQnode *tail; - STaosQueue *next; // for queue set - STaosQset *qset; // for queue set - void *ahandle; // for queue set - FProcessItem itemFp; - FProcessItems itemsFp; + int32_t threadId; + STaosQnode * head; + STaosQnode * tail; + STaosQueue * next; // for queue set + STaosQset * qset; // for queue set + void * ahandle; // for queue set + FItem itemFp; + FItems itemsFp; pthread_mutex_t mutex; } STaosQueue; typedef struct STaosQset { - STaosQueue *head; - STaosQueue *current; + STaosQueue * head; + STaosQueue * current; pthread_mutex_t mutex; int32_t numOfQueues; int32_t numOfItems; @@ -56,19 +57,23 @@ typedef struct STaosQall { } STaosQall; STaosQueue *taosOpenQueue() { - STaosQueue *queue = calloc(sizeof(STaosQueue), 1); + STaosQueue *queue = calloc(1, sizeof(STaosQueue)); if (queue == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pthread_mutex_init(&queue->mutex, NULL); + if (pthread_mutex_init(&queue->mutex, NULL) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - uTrace("queue:%p is opened", queue); + queue->threadId = -1; + uDebug("queue:%p is opened", queue); return queue; } -void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsFp) { +void taosSetQueueFp(STaosQueue *queue, FItem itemFp, FItems itemsFp) { if (queue == NULL) return; queue->itemFp = itemFp; queue->itemsFp = itemsFp; @@ -77,7 +82,7 @@ void taosSetQueueFp(STaosQueue *queue, FProcessItem itemFp, FProcessItems itemsF void taosCloseQueue(STaosQueue *queue) { if (queue == NULL) return; STaosQnode *pTemp; - STaosQset *qset; + STaosQset * qset; pthread_mutex_lock(&queue->mutex); STaosQnode *pNode = queue->head; @@ -85,7 +90,9 @@ void taosCloseQueue(STaosQueue *queue) { qset = queue->qset; pthread_mutex_unlock(&queue->mutex); - if (queue->qset) taosRemoveFromQset(qset, queue); + if (queue->qset) { + taosRemoveFromQset(qset, queue); + } while (pNode) { pTemp = pNode; @@ -96,7 +103,7 @@ void taosCloseQueue(STaosQueue *queue) { pthread_mutex_destroy(&queue->mutex); free(queue); - uTrace("queue:%p is closed", queue); + uDebug("queue:%p is closed", queue); } bool taosQueueEmpty(STaosQueue *queue) { @@ -120,19 +127,23 @@ int32_t taosQueueSize(STaosQueue *queue) { } void *taosAllocateQitem(int32_t size) { - STaosQnode *pNode = (STaosQnode *)calloc(sizeof(STaosQnode) + size, 1); + STaosQnode *pNode = calloc(1, sizeof(STaosQnode) + size); + + if (pNode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - if (pNode == NULL) return NULL; uTrace("item:%p, node:%p is allocated", pNode->item, pNode); return (void *)pNode->item; } -void taosFreeQitem(void *param) { - if (param == NULL) return; +void taosFreeQitem(void *pItem) { + if (pItem == NULL) return; - char *temp = (char *)param; + char *temp = pItem; temp -= sizeof(STaosQnode); - uTrace("item:%p, node:%p is freed", param, temp); + uTrace("item:%p, node:%p is freed", pItem, temp); free(temp); } @@ -175,7 +186,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { queue->numOfItems--; if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, 1); code = 1; - uDebug("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); + uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); } pthread_mutex_unlock(&queue->mutex); @@ -183,7 +194,7 @@ int32_t taosReadQitem(STaosQueue *queue, void **ppItem) { return code; } -STaosQall *taosAllocateQall() { return calloc(sizeof(STaosQall), 1); } +STaosQall *taosAllocateQall() { return calloc(1, sizeof(STaosQall)); } void taosFreeQall(STaosQall *qall) { free(qall); } @@ -238,7 +249,7 @@ int32_t taosGetQitem(STaosQall *qall, void **ppItem) { void taosResetQitems(STaosQall *qall) { qall->current = qall->start; } STaosQset *taosOpenQset() { - STaosQset *qset = (STaosQset *)calloc(sizeof(STaosQset), 1); + STaosQset *qset = calloc(sizeof(STaosQset), 1); if (qset == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -247,7 +258,7 @@ STaosQset *taosOpenQset() { pthread_mutex_init(&qset->mutex, NULL); tsem_init(&qset->sem, 0, 0); - uTrace("qset:%p is opened", qset); + uDebug("qset:%p is opened", qset); return qset; } @@ -268,7 +279,7 @@ void taosCloseQset(STaosQset *qset) { pthread_mutex_destroy(&qset->mutex); tsem_destroy(&qset->sem); free(qset); - uTrace("qset:%p is closed", qset); + uDebug("qset:%p is closed", qset); } // tsem_post 'qset->sem', so that reader threads waiting for it @@ -338,12 +349,12 @@ void taosRemoveFromQset(STaosQset *qset, STaosQueue *queue) { pthread_mutex_unlock(&qset->mutex); - uTrace("queue:%p is removed from qset:%p", queue, qset); + uDebug("queue:%p is removed from qset:%p", queue, qset); } int32_t taosGetQueueNumber(STaosQset *qset) { return qset->numOfQueues; } -int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FProcessItem *itemFp) { +int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp) { STaosQnode *pNode = NULL; int32_t code = 0; @@ -365,6 +376,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FP *ppItem = pNode->item; if (ahandle) *ahandle = queue->ahandle; if (itemFp) *itemFp = queue->itemFp; + queue->head = pNode->next; if (queue->head == NULL) queue->tail = NULL; queue->numOfItems--; @@ -382,7 +394,7 @@ int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FP return code; } -int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FProcessItems *itemsFp) { +int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp) { STaosQueue *queue; int32_t code = 0; @@ -411,7 +423,9 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand queue->tail = NULL; queue->numOfItems = 0; atomic_sub_fetch_32(&qset->numOfItems, qall->numOfItems); - for (int32_t j = 1; j < qall->numOfItems; ++j) tsem_wait(&qset->sem); + for (int32_t j = 1; j < qall->numOfItems; ++j) { + tsem_wait(&qset->sem); + } } pthread_mutex_unlock(&queue->mutex); @@ -423,6 +437,65 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand return code; } +int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp, int32_t threadId) { + STaosQnode *pNode = NULL; + int32_t code = -1; + + tsem_wait(&qset->sem); + + pthread_mutex_lock(&qset->mutex); + + for (int32_t i = 0; i < qset->numOfQueues; ++i) { + if (qset->current == NULL) qset->current = qset->head; + STaosQueue *queue = qset->current; + if (queue) qset->current = queue->next; + if (queue == NULL) break; + if (queue->head == NULL) continue; + if (queue->threadId != -1 && queue->threadId != threadId) { + code = 0; + continue; + } + + pthread_mutex_lock(&queue->mutex); + + if (queue->head) { + pNode = queue->head; + pNode->queue = queue; + queue->threadId = threadId; + *ppItem = pNode->item; + + if (ahandle) *ahandle = queue->ahandle; + if (itemFp) *itemFp = queue->itemFp; + + queue->head = pNode->next; + if (queue->head == NULL) queue->tail = NULL; + queue->numOfItems--; + atomic_sub_fetch_32(&qset->numOfItems, 1); + code = 1; + uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); + } + + pthread_mutex_unlock(&queue->mutex); + if (pNode) break; + } + + pthread_mutex_unlock(&qset->mutex); + + return code; +} + +void taosResetQsetThread(STaosQset *qset, void *pItem) { + if (pItem == NULL) return; + STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode)); + + pthread_mutex_lock(&qset->mutex); + pNode->queue->threadId = -1; + for (int32_t i = 0; i < pNode->queue->numOfItems; ++i) { + tsem_post(&qset->sem); + } + pthread_mutex_unlock(&qset->mutex); +} + int32_t taosGetQueueItemsNumber(STaosQueue *queue) { if (!queue) return 0; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index ed74041712ce0022eb66ac5c9331b88fda961222..97440f8dae3ccdbcaa9dc75a8e2f5c3b2da02cf7 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -14,38 +14,46 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" -#include "tqueue.h" #include "tworker.h" +#include "taoserror.h" +#include "ulog.h" -typedef void* (*ThreadFp)(void *param); +typedef void *(*ThreadFp)(void *param); -int32_t tWorkerInit(SWorkerPool *pool) { +int32_t tQWorkerInit(SQWorkerPool *pool) { pool->qset = taosOpenQset(); - pool->workers = calloc(sizeof(SWorker), pool->max); - pthread_mutex_init(&pool->mutex, NULL); - for (int i = 0; i < pool->max; ++i) { - SWorker *worker = pool->workers + i; + pool->workers = calloc(sizeof(SQWorker), pool->max); + if (pool->workers == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (pthread_mutex_init(&pool->mutex, NULL)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < pool->max; ++i) { + SQWorker *worker = pool->workers + i; worker->id = i; worker->pool = pool; } - uInfo("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); + uDebug("worker:%s is initialized, min:%d max:%d", pool->name, pool->min, pool->max); return 0; } -void tWorkerCleanup(SWorkerPool *pool) { - for (int i = 0; i < pool->max; ++i) { - SWorker *worker = pool->workers + i; +void tQWorkerCleanup(SQWorkerPool *pool) { + for (int32_t i = 0; i < pool->max; ++i) { + SQWorker *worker = pool->workers + i; if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { taosQsetThreadResume(pool->qset); } } - for (int i = 0; i < pool->max; ++i) { - SWorker *worker = pool->workers + i; + for (int32_t i = 0; i < pool->max; ++i) { + SQWorker *worker = pool->workers + i; if (worker == NULL) continue; if (taosCheckPthreadValid(worker->thread)) { pthread_join(worker->thread, NULL); @@ -56,15 +64,15 @@ void tWorkerCleanup(SWorkerPool *pool) { taosCloseQset(pool->qset); pthread_mutex_destroy(&pool->mutex); - uInfo("worker:%s is closed", pool->name); + uDebug("worker:%s is closed", pool->name); } -static void *tWorkerThreadFp(SWorker *worker) { - SWorkerPool *pool = worker->pool; - FProcessItem fp = NULL; +static void *tQWorkerThreadFp(SQWorker *worker) { + SQWorkerPool *pool = worker->pool; + FItem fp = NULL; - void *msg = NULL; - void *ahandle = NULL; + void * msg = NULL; + void * ahandle = NULL; int32_t code = 0; taosBlockSIGPIPE(); @@ -77,7 +85,7 @@ static void *tWorkerThreadFp(SWorker *worker) { break; } - if (fp) { + if (fp != NULL) { (*fp)(ahandle, msg); } } @@ -85,11 +93,12 @@ static void *tWorkerThreadFp(SWorker *worker) { return NULL; } -STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) { +STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, ThreadFp threadFp) { pthread_mutex_lock(&pool->mutex); STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { pthread_mutex_unlock(&pool->mutex); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -99,14 +108,18 @@ STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) // spawn a thread to process queue if (pool->num < pool->max) { do { - SWorker *worker = pool->workers + pool->num; + SQWorker *worker = pool->workers + pool->num; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWorkerThreadFp, worker) != 0) { + if (pthread_create(&worker->thread, &thAttr, threadFp, worker) != 0) { uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); + taosCloseQueue(queue); + terrno = TSDB_CODE_OUT_OF_MEMORY; + queue = NULL; + break; } pthread_attr_destroy(&thAttr); @@ -121,19 +134,73 @@ STaosQueue *tWorkerAllocQueue(SWorkerPool *pool, void *ahandle, FProcessItem fp) return queue; } -void tWorkerFreeQueue(SWorkerPool *pool, STaosQueue *queue) { +STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { + return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tQWorkerThreadFp); +} + +void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } -int32_t tMWorkerInit(SMWorkerPool *pool) { +int32_t tFWorkerInit(SFWorkerPool *pool) { return tQWorkerInit((SQWorkerPool *)pool); } + +void tFWorkerCleanup(SFWorkerPool *pool) { tQWorkerCleanup(pool); } + +static void *tFWorkerThreadFp(SQWorker *worker) { + SQWorkerPool *pool = worker->pool; + + FItem fp = NULL; + void * msg = NULL; + void * ahandle = NULL; + int32_t code = 0; + + taosBlockSIGPIPE(); + setThreadName(pool->name); + uDebug("worker:%s:%d is running", pool->name, worker->id); + + while (1) { + code = taosReadQitemFromQsetByThread(pool->qset, (void **)&msg, &ahandle, &fp, worker->id); + + if (code < 0) { + uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset); + break; + } else if (code == 0) { + // uTrace("worker:%s:%d qset:%p, got no message and continue", pool->name, worker->id, pool->qset); + continue; + } + + if (fp != NULL) { + (*fp)(ahandle, msg); + } + + taosResetQsetThread(pool->qset, msg); + } + + return NULL; +} + +STaosQueue *tFWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { + return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tFWorkerThreadFp); +} + +void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue) { tQWorkerFreeQueue(pool, queue); } + +int32_t tWWorkerInit(SWWorkerPool *pool) { pool->nextId = 0; - pool->workers = calloc(sizeof(SMWorker), pool->max); - if (pool->workers == NULL) return -1; + pool->workers = calloc(sizeof(SWWorker), pool->max); + if (pool->workers == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (pthread_mutex_init(&pool->mutex, NULL) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } - pthread_mutex_init(&pool->mutex, NULL); for (int32_t i = 0; i < pool->max; ++i) { - SMWorker *worker = pool->workers + i; + SWWorker *worker = pool->workers + i; worker->id = i; worker->qall = NULL; worker->qset = NULL; @@ -144,16 +211,18 @@ int32_t tMWorkerInit(SMWorkerPool *pool) { return 0; } -void tMWorkerCleanup(SMWorkerPool *pool) { +void tWWorkerCleanup(SWWorkerPool *pool) { for (int32_t i = 0; i < pool->max; ++i) { - SMWorker *worker = pool->workers + i; + SWWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { - if (worker->qset) taosQsetThreadResume(worker->qset); + if (worker->qset) { + taosQsetThreadResume(worker->qset); + } } } for (int32_t i = 0; i < pool->max; ++i) { - SMWorker *worker = pool->workers + i; + SWWorker *worker = pool->workers + i; if (taosCheckPthreadValid(worker->thread)) { pthread_join(worker->thread, NULL); taosFreeQall(worker->qall); @@ -167,12 +236,12 @@ void tMWorkerCleanup(SMWorkerPool *pool) { uInfo("worker:%s is closed", pool->name); } -static void *tWriteWorkerThreadFp(SMWorker *worker) { - SMWorkerPool *pool = worker->pool; - FProcessItems fp = NULL; +static void *tWWorkerThreadFp(SWWorker *worker) { + SWWorkerPool *pool = worker->pool; + FItems fp = NULL; - void *msg = NULL; - void *ahandle = NULL; + void * msg = NULL; + void * ahandle = NULL; int32_t numOfMsgs = 0; int32_t qtype = 0; @@ -187,7 +256,7 @@ static void *tWriteWorkerThreadFp(SMWorker *worker) { break; } - if (fp) { + if (fp != NULL) { (*fp)(ahandle, worker->qall, numOfMsgs); } } @@ -195,13 +264,14 @@ static void *tWriteWorkerThreadFp(SMWorker *worker) { return NULL; } -STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems fp) { +STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp) { pthread_mutex_lock(&pool->mutex); - SMWorker *worker = pool->workers + pool->nextId; + SWWorker *worker = pool->workers + pool->nextId; STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { pthread_mutex_unlock(&pool->mutex); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } @@ -221,17 +291,19 @@ STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems taosCloseQset(worker->qset); taosCloseQueue(queue); pthread_mutex_unlock(&pool->mutex); + terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWriteWorkerThreadFp, worker) != 0) { + if (pthread_create(&worker->thread, &thAttr, (ThreadFp)tWWorkerThreadFp, worker) != 0) { uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); taosFreeQall(worker->qall); taosCloseQset(worker->qset); taosCloseQueue(queue); + terrno = TSDB_CODE_OUT_OF_MEMORY; queue = NULL; } else { uDebug("worker:%s:%d is launched, max:%d", pool->name, worker->id, pool->max); @@ -250,7 +322,7 @@ STaosQueue *tMWorkerAllocQueue(SMWorkerPool *pool, void *ahandle, FProcessItems return queue; } -void tMWorkerFreeQueue(SMWorkerPool *pool, STaosQueue *queue) { +void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index 4b6311022d997d87ee09c3b379e7f8e9c28c7dad..383a00232a451f3288fbb0bf629191ce4488f80a 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -13,12 +13,12 @@ IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c) ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(utilTest util common os gtest pthread gcov) + TARGET_LINK_LIBRARIES(utilTest util common os gtest pthread) LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/cacheTest.cpp) LIST(APPEND SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/hashTest.cpp) ADD_EXECUTABLE(hashTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(hashTest util common os gtest pthread gcov) + TARGET_LINK_LIBRARIES(hashTest util common os gtest pthread) LIST(APPEND BIN_SRC ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c) ADD_EXECUTABLE(trefTest ${BIN_SRC}) diff --git a/tests/script/sh/dockerbuilder/addDnodeToCluster.sh b/tests/script/sh/dockerbuilder/addDnodeToCluster.sh new file mode 100755 index 0000000000000000000000000000000000000000..1886632350bd539bd22f9423bf0fe1d6ae73e191 --- /dev/null +++ b/tests/script/sh/dockerbuilder/addDnodeToCluster.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# deploy test cluster + +set -e +#set -x + +dnodeNumber=1 +serverPort=6030 + +while getopts "hn:p:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + p) + serverPort=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +for ((i=2; i<=${dnodeNumber}; i++)); do + taos -s "create dnode node${i} port ${serverPort};" ||: + echo "create dnode node${i} port ${serverPort};" +done + + + + diff --git a/tests/script/sh/dockerbuilder/addFqdnToHost.sh b/tests/script/sh/dockerbuilder/addFqdnToHost.sh new file mode 100755 index 0000000000000000000000000000000000000000..4d613d1dd0fad834b6262123d04a5a1e9ec39297 --- /dev/null +++ b/tests/script/sh/dockerbuilder/addFqdnToHost.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# +# deploy test cluster + +set -e +#set -x + +dnodeNumber=1 +subnet="172.33.0.0/16" + +while getopts "hn:s:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + s) + subnet=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + echo " -s [ subnet] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +addFqdnToHosts() { + index=$1 + ipPrefix=$2 + let ipIndex=index+1 + echo "${ipPrefix}.${ipIndex} node${i}" >> /etc/hosts +} + +ipPrefix=${subnet%.*} +for ((i=1; i<=${dnodeNumber}; i++)); do + addFqdnToHosts ${i} ${ipPrefix} +done + + + + diff --git a/tests/script/sh/dockerbuilder/clientCfg/taos.cfg b/tests/script/sh/dockerbuilder/clientCfg/taos.cfg new file mode 100644 index 0000000000000000000000000000000000000000..491216d5f67adaef64883ab26e9193d907b8a2ce --- /dev/null +++ b/tests/script/sh/dockerbuilder/clientCfg/taos.cfg @@ -0,0 +1,4 @@ +debugFlag 131 +firstEp node1:6030 +asyncLog 0 +telemetryReporting 0 diff --git a/tests/script/sh/dockerbuilder/createComposeYml.sh b/tests/script/sh/dockerbuilder/createComposeYml.sh new file mode 100755 index 0000000000000000000000000000000000000000..494e79318bcea095534a7ff6044da0a42636a0e4 --- /dev/null +++ b/tests/script/sh/dockerbuilder/createComposeYml.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# +# create docker-compose.yml + +set -e +#set -x + +# set parameters by default value +composeYmlFile="./docker-compose.yml" +dnodeNumber=1 +subnet="172.33.0.0/16" + +while getopts "hn:f:s:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + f) + composeYmlFile=$(echo $OPTARG) + ;; + s) + subnet=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + echo " -f [ yml file] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "dnodeNumber=${dnodeNumber} composeYmlFile=${composeYmlFile}" + +createFirstSection() { + ymlFile=$1 + + echo "version: '3.7'" > ${ymlFile} + echo "" >> ${ymlFile} + echo "x-node: &x-node" >> ${ymlFile} + echo " build:" >> ${ymlFile} + echo " context: ." >> ${ymlFile} + echo " dockerfile: ./tdserver/Dockerfile" >> ${ymlFile} + echo " args:" >> ${ymlFile} + echo " - PACKAGE=TDengine-server-3.0.0.0-Linux-x64.tar.gz" >> ${ymlFile} + echo " - EXTRACTDIR=TDengine-server-3.0.0.0" >> ${ymlFile} + echo " image: 'tdengine:3.0.0.0'" >> ${ymlFile} + echo " container_name: 'node1'" >> ${ymlFile} + echo " privileged: true" >> ${ymlFile} + echo " cap_add:" >> ${ymlFile} + echo " - ALL" >> ${ymlFile} + echo " stdin_open: true" >> ${ymlFile} + echo " tty: true" >> ${ymlFile} + echo " environment:" >> ${ymlFile} + echo ' TZ: "Asia/Shanghai"' >> ${ymlFile} + echo ' command: >' >> ${ymlFile} + echo ' sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime &&' >> ${ymlFile} + echo ' echo $TZ > /etc/timezone &&' >> ${ymlFile} + echo ' exec sysctl -w kernel.core_pattern=/corefile/core-%e-%p"' >> ${ymlFile} + echo " restart: always" >> ${ymlFile} + echo " hostname: node1" >> ${ymlFile} + echo " command: taosd" >> ${ymlFile} + echo " deploy:" >> ${ymlFile} + echo " resources:" >> ${ymlFile} + echo " limits:" >> ${ymlFile} + echo ' cpus: "2.00"' >> ${ymlFile} + echo " memory: 4G" >> ${ymlFile} + echo " reservations:" >> ${ymlFile} + echo ' cpus: "1.00"' >> ${ymlFile} + echo " memory: 500M" >> ${ymlFile} + echo " volumes:" >> ${ymlFile} + echo " - /etc/localtime:/etc/localtime:ro" >> ${ymlFile} + echo ' - $PWD:/work' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/data:/var/lib/taos' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/log:/var/log/taos' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/cfg:/etc/taos' >> ${ymlFile} + echo ' - $PWD/storage/dnode1/core:/corefile' >> ${ymlFile} + echo "" >> ${ymlFile} + echo "networks:" >> ${ymlFile} + echo " tdnet:" >> ${ymlFile} + echo " ipam:" >> ${ymlFile} + echo " driver: default" >> ${ymlFile} + echo " config:" >> ${ymlFile} + echo " - subnet: ${subnet}" >> ${ymlFile} + echo "" >> ${ymlFile} + echo "services:" >> ${ymlFile} +} + +createSingleDnodesCfg() { + ymlFile=$1 + index=$2 + + ipPrefix=${subnet%.*} + + let ipIndex=index+1 + + echo " node${index}:" >> ${ymlFile} + echo " <<: *x-node" >> ${ymlFile} + echo " container_name: 'node${index}'" >> ${ymlFile} + echo " hostname: node${index}" >> ${ymlFile} + echo " networks:" >> ${ymlFile} + echo " tdnet:" >> ${ymlFile} + echo " ipv4_address: ${ipPrefix}.${ipIndex}" >> ${ymlFile} + echo " volumes:" >> ${ymlFile} + echo " - /etc/localtime:/etc/localtime:ro" >> ${ymlFile} + echo " - \$PWD:/work" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/data:/var/lib/taos" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/log:/var/log/taos" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/cfg:/etc/taos" >> ${ymlFile} + echo " - \$PWD/storage/dnode${index}/core:/corefile" >> ${ymlFile} + echo "" >> ${ymlFile} +} + +createDnodesOfDockerComposeYml() { + ymlFile=$1 + dnodeNumber=$2 + + for ((i=1; i<=${dnodeNumber}; i++)); do + createSingleDnodesCfg ${ymlFile} ${i} + done +} + +######################################################################################## +############################### main process ########################################## + +createFirstSection ${composeYmlFile} +createDnodesOfDockerComposeYml ${composeYmlFile} ${dnodeNumber} + +echo "====create docker-compose.yml end====" +echo " " + + diff --git a/tests/script/sh/dockerbuilder/createDnodeStorage.sh b/tests/script/sh/dockerbuilder/createDnodeStorage.sh new file mode 100755 index 0000000000000000000000000000000000000000..4e9f8e5121c92f0476cee0f47a09f6fe38eb2f5c --- /dev/null +++ b/tests/script/sh/dockerbuilder/createDnodeStorage.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# +# setup test environment + +set -e +#set -x + +# set parameters by default value +dataRootDir="/data/dockerbuilder/storage" +dnodeNumber=1 +firstEp="node1:6030" + +while getopts "hn:r:f:" arg +do + case $arg in + n) + dnodeNumber=$(echo $OPTARG) + ;; + f) + firstEp=$(echo $OPTARG) + ;; + r) + dataRootDir=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -n [ dnode number] " + echo " -n [ dnode number] " + echo " -f [ first ep] " + echo " -r [ data root dir] " + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "dnodeNumber=${dnodeNumber} dataRootDir=${dataRootDir} firstEp=${firstEp}" + +createTaosCfg() { + cfgFile=$1/cfg/taos.cfg + firstEp=$2 + fqdn=$3 + + echo "debugFlag 131" > ${cfgFile} + echo "firstEp ${firstEp}" >> ${cfgFile} + #echo "dataDir ${dataDir}" >> ${cfgFile} + #echo "logDir ${logDir}" >> ${cfgFile} + echo "fqdn ${fqdn}" >> ${cfgFile} + + echo "supportVnodes 1024" >> ${cfgFile} + echo "asyncLog 0" >> ${cfgFile} + echo "telemetryReporting 0" >> ${cfgFile} +} + +createDnodesDataDir() { + if [ -d ${dataRootDir} ]; then + rm -rf ${dataRootDir}/* + else + echo "${dataRootDir} not exist" + exit 1 + fi + + dnodeNumber=$1 + firstEp=$2 + + serverPort=${startPort} + for ((i=1; i<=${dnodeNumber}; i++)); do + mkdir -p ${dataRootDir}/dnode${i}/cfg + mkdir -p ${dataRootDir}/dnode${i}/log + mkdir -p ${dataRootDir}/dnode${i}/data + mkdir -p ${dataRootDir}/dnode${i}/core + + createTaosCfg ${dataRootDir}/dnode${i} ${firstEp} node${i} + done +} + +######################################################################################## +############################### main process ########################################## + +## create director and taos.cfg for all dnode +createDnodesDataDir ${dnodeNumber} ${firstEp} + +echo "====create end====" +echo " " + + diff --git a/tests/script/sh/dockerbuilder/tdserver/Dockerfile b/tests/script/sh/dockerbuilder/tdserver/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..7757aa68c783f1fd9b8359b3ce736da06585886f --- /dev/null +++ b/tests/script/sh/dockerbuilder/tdserver/Dockerfile @@ -0,0 +1,42 @@ +FROM centos:8 AS builder + +ARG PACKAGE=TDengine-server-3.0.0.0-Linux-x64.tar.gz +ARG EXTRACTDIR=TDengine-server-3.0.0.0 + +WORKDIR /root + +COPY ${PACKAGE} . +COPY tmux.conf . +COPY addDnodeToCluster.sh . + +RUN tar -zxf ${PACKAGE} +RUN mv ${EXTRACTDIR}/* ./ + +FROM centos:8 + +WORKDIR /root + +RUN yum install -y glibc-langpack-en dmidecode gdb +RUN yum install -y tmux net-tools +RUN yum install -y sysstat +RUN yum install -y vim +RUN echo 'alias ll="ls -l --color=auto"' >> ~/.bashrc + +COPY --from=builder /root/taosd /usr/bin +COPY --from=builder /root/taos /usr/bin +COPY --from=builder /root/create_table /usr/bin +COPY --from=builder /root/tmux.conf /root/.tmux.conf +#COPY --from=builder /root/addDnodeToCluster.sh /root/addDnodeToCluster.sh + +#COPY --from=builder /root/cfg/taos.cfg /etc/taos/ +COPY --from=builder /root/lib/* /usr/lib/ + +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" +ENV LC_CTYPE=en_US.UTF-8 +ENV LANG=en_US.UTF-8 + +EXPOSE 6030-6042/tcp 6060/tcp 6030-6039/udp + +# VOLUME [ "/var/lib/taos", "/var/log/taos", "/etc/taos" ] + +CMD [ "bash" ] diff --git a/tests/script/sh/dockerbuilder/tmux.conf b/tests/script/sh/dockerbuilder/tmux.conf new file mode 100644 index 0000000000000000000000000000000000000000..d1f1801f4a323da39244cd9e1bdb227e2ff9ec05 --- /dev/null +++ b/tests/script/sh/dockerbuilder/tmux.conf @@ -0,0 +1,115 @@ +unbind C-b +# remap prefix to Control + z +set -g prefix C-a +# bind 'C-z C-z' to type 'C-z' +bind C-a send-prefix + + +# 0 is too far from ` ;) +set -g base-index 1 + +# Automatically set window title +set-window-option -g automatic-rename on +set-option -g set-titles on + +#set -g default-terminal screen-256color +set -g status-keys vi +set -g history-limit 10000 + +setw -g mode-keys vi +#setw -g mode-mouse on +setw -g monitor-activity on +set -g mouse on + +bind-key v split-window -h -c "#{pane_current_path}" +bind-key s split-window -v -c "#{pane_current_path}" +bind-key c new-window -c "#{pane_current_path}" + +bind-key -n C-S-Left swap-window -t -1 +bind-key -n C-S-Right swap-window -t +1 + +# bind-key J resize-pane -D 5 +# bind-key K resize-pane -U 5 +# bind-key H resize-pane -L 5 +# bind-key L resize-pane -R 5 + +# bind-key M-j resize-pane -D +# bind-key M-k resize-pane -U +# bind-key M-h resize-pane -L +# bind-key M-l resize-pane -R + +# Vim style pane selection +bind h select-pane -L +bind j select-pane -D +bind k select-pane -U +bind l select-pane -R +bind K kill-pane -a + +# Use Alt-vim keys without prefix key to switch panes +bind -n M-h select-pane -L +bind -n M-j select-pane -D +bind -n M-k select-pane -U +bind -n M-l select-pane -R +# bind -n M-h select-pane -L \; resize-pane -Z +# bind -n M-j select-pane -D \; resize-pane -Z +# bind -n M-k select-pane -U \; resize-pane -Z +# bind -n M-l select-pane -R \; resize-pane -Z + +# # Use Alt-arrow keys without prefix key to switch panes +bind -n M-Left select-pane -L +bind -n M-Right select-pane -R +bind -n M-Up select-pane -U +bind -n M-Down select-pane -D + +# Shift arrow to switch windows +bind -n C-Left previous-window +bind -n C-Right next-window + +bind -n M-n previous-window +bind -n M-p nejt-window + +# Map Alt-z to zoom in and out pane +bind -n M-z resize-pane -Z +bind -n M-q resize-pane -Z + +# # Use v to trigger selection +bind-key -T copy-mode-vi v send-keys -X begin-selection +# +# # Use y to yank current selection +bind-key -T copy-mode-vi y send-keys -X copy-selection-and-cancel + +# No delay for escape key press +set -sg escape-time 0 + +# THEME +set -g status-style bg=black,fg=white +# set -g status-bg black +# set -g status-fg white +set -g window-status-current-style bg=green,fg=black,bold +set -g pane-active-border-style fg=green,bg=black +set -g status-interval 60 +set -g status-left-length 30 +set -g status-left '#[fg=green](#S) #(whoami)' +set -g status-right '#[fg=yellow]#(cut -d " " -f 1-3 /proc/loadavg)#[default] #[fg=white]%H:%M#[default]' + +# synchronouse windows +# setw synchronize-panes + +# pane movement +bind-key b command-prompt -p "join pane from:" "join-pane -s ':%%'" +bind-key a command-prompt -p "send pane to:" "join-pane -t ':%%'" +# # for vim +# # Smart pane switching with awareness of Vim splits. +# # See: https://github.com/christoomey/vim-tmux-navigator +# is_vim="ps -o state= -o comm= -t '#{pane_tty}' \ +# | grep -iqE '^[^TXZ ]+ +(\\S+\\/)?g?(view|n?vim?x?)(diff)?$'" +# bind-key -n C-h if-shell "$is_vim" "send-keys C-h" "select-pane -L" +# bind-key -n C-j if-shell "$is_vim" "send-keys C-j" "select-pane -D" +# bind-key -n C-k if-shell "$is_vim" "send-keys C-k" "select-pane -U" +# bind-key -n C-l if-shell "$is_vim" "send-keys C-l" "select-pane -R" +# bind-key -n C-\ if-shell "$is_vim" "send-keys C-\\" "select-pane -l" +# bind-key -T copy-mode-vi C-h select-pane -L +# bind-key -T copy-mode-vi C-j select-pane -D +# bind-key -T copy-mode-vi C-k select-pane -U +# bind-key -T copy-mode-vi C-l select-pane -R +# bind-key -T copy-mode-vi C-\ select-pane -l diff --git a/tests/script/sh/massiveTable/deployCluster.sh b/tests/script/sh/massiveTable/deployCluster.sh index beb324321d52e7669619e0b17b5726a194c22f97..503f3ccae29d3b075855929d1ee8d6fe0666feee 100755 --- a/tests/script/sh/massiveTable/deployCluster.sh +++ b/tests/script/sh/massiveTable/deployCluster.sh @@ -5,10 +5,12 @@ set -e #set -x +fqdn=`hostname` + masterDnode=slave dataRootDir="/data" -firstEp="trd02:7000" -startPort=7000 +firstEp="${fqdn}:6030" +startPort=6030 dnodeNumber=1 updateSrc=no @@ -52,23 +54,17 @@ done curr_dir=$(readlink -f "$(dirname "$0")") echo $curr_dir -${curr_dir}/cleanCluster.sh -r "/data" -#${curr_dir}/cleanCluster.sh -r "/data2" +${curr_dir}/cleanCluster.sh -r ${dataRootDir} if [[ "${updateSrc}" == "yes" ]]; then ${curr_dir}/compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0" fi -${curr_dir}/setupDnodes.sh -r "/data" -n ${dnodeNumber} -f ${firstEp} -p 7000 -#${curr_dir}/setupDnodes.sh -r "/data2" -n ${dnodeNumber} -f ${firstEp} -p 8000 +${curr_dir}/setupDnodes.sh -r ${dataRootDir} -n ${dnodeNumber} -f ${firstEp} -p ${startPort} if [[ "${masterDnode}" == "master" ]]; then - # create all dnode into cluster - #taos -s "create dnode trd02 port 8000;" - taos -s "create dnode trd03 port 7000;" - #taos -s "create dnode trd03 port 8000;" - taos -s "create dnode trd04 port 7000;" - #taos -s "create dnode trd04 port 8000;" + taos -s "create dnode trd03 port 6030;" + taos -s "create dnode trd04 port 6030;" fi diff --git a/tests/script/sh/massiveTable/setupDnodes.sh b/tests/script/sh/massiveTable/setupDnodes.sh index 37fdbaf78457e2e57a9761652cf96eb91b6f0151..22c325cd893395c5191927fb8f11e7edabca18a5 100755 --- a/tests/script/sh/massiveTable/setupDnodes.sh +++ b/tests/script/sh/massiveTable/setupDnodes.sh @@ -13,10 +13,12 @@ set -e # -r [ dnode root dir] # set parameters by default value +fqdn=`hostname` + enviMode=new dataRootDir="/data" -firstEp="localhost:7000" -startPort=7000 +firstEp="${fqdn}:6030" +startPort=6030 dnodeNumber=1