diff --git a/Jenkinsfile b/Jenkinsfile index 2558df777b5698de59093316da0e18407a160d77..5faa02d0c3162c9b65aa1e9d5eac44de5118a582 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,7 +42,6 @@ def pre_test(){ killall -9 gdb || echo "no gdb running" killall -9 python3.8 || echo "no python program running" cd ${WKC} - git reset --hard HEAD~10 >/dev/null ''' script { if (env.CHANGE_TARGET == 'master') { @@ -75,11 +74,9 @@ def pre_test(){ git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD - git clean -dfx - - git clean -dfx export TZ=Asia/Harbin date + rm -rf debug mkdir debug cd debug cmake .. > /dev/null diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 11a38a3742e049d11a8f06bca7f429d896c91e37..aef7d5711fb1b73d2542ebe34b9c77c636827807 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -50,9 +50,8 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONSUME, "mq-consume" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_QUERY, "mq-query" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_CONNECT, "mq-connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_DISCONNECT, "mq-disconnect" ) -TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET, "mq-set" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_MQ_SET_CUR, "mq-set-cur" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_RSP_READY, "rsp-ready" ) - // message from client to mnode TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CONNECT, "connect" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CREATE_ACCT, "create-acct" ) @@ -225,6 +224,7 @@ typedef struct SBuildUseDBInput { int32_t vgVersion; } SBuildUseDBInput; + #pragma pack(push, 1) // null-terminated string instead of char array to avoid too many memory consumption in case of more than 1M tableMeta @@ -343,7 +343,7 @@ typedef struct { typedef struct { char tableFname[TSDB_TABLE_FNAME_LEN]; char db[TSDB_FULL_DB_NAME_LEN]; - int16_t type; /* operation type */ + int16_t type; /* operation type */ int16_t numOfCols; /* number of schema */ int32_t tagValLen; SSchema schema[]; @@ -545,8 +545,8 @@ typedef struct { int32_t sqlstrLen; // sql query string int32_t prevResultLen; // previous result length int32_t numOfOperator; - int32_t tableScanOperator; // table scan operator. -1 means no scan operator - int32_t udfNum; // number of udf function + int32_t tableScanOperator;// table scan operator. -1 means no scan operator + int32_t udfNum; // number of udf function int32_t udfContentOffset; int32_t udfContentLen; SColumnInfo tableCols[]; @@ -591,8 +591,8 @@ typedef struct { int32_t daysToKeep0; int32_t daysToKeep1; int32_t daysToKeep2; - int32_t minRowsPerFileBlock; - int32_t maxRowsPerFileBlock; + int32_t minRows; + int32_t maxRows; int32_t commitTime; int32_t fsyncPeriod; int8_t walLevel; @@ -706,7 +706,7 @@ typedef struct { SVnodeLoad data[]; } SVnodeLoads; -typedef struct SStatusMsg { +typedef struct { int32_t sver; int32_t dnodeId; int32_t clusterId; @@ -756,6 +756,7 @@ typedef struct { int32_t dnodeId; char db[TSDB_FULL_DB_NAME_LEN]; uint64_t dbUid; + int32_t vgVersion; int32_t cacheBlockSize; int32_t totalBlocks; int32_t daysPerFile; @@ -1005,27 +1006,35 @@ typedef struct { // mq related typedef struct { + } SMqConnectReq; typedef struct { + } SMqConnectRsp; typedef struct { + } SMqDisconnectReq; typedef struct { + } SMqDisconnectRsp; typedef struct { + } SMqAckReq; typedef struct { + } SMqAckRsp; typedef struct { + } SMqResetReq; typedef struct { + } SMqResetRsp; // mq related end diff --git a/include/common/tmsgtype.h b/include/common/tmsgtype.h index 1fb10ae15b692d21f8a07937393399fe49664ce3..09765031155679accc6f6d4043e23b3b89409135 100644 --- a/include/common/tmsgtype.h +++ b/include/common/tmsgtype.h @@ -35,7 +35,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SELECT, "select" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_FETCH, "fetch" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_INSERT, "insert" ) - TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAGS_VAL, "update-tag-val" ) + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_UPDATE_TAG_VAL, "update-tag-val" ) // the SQL below is for mgmt node TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" ) @@ -54,7 +54,7 @@ enum { TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_TABLE, "alter-table" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_ALTER_DB, "alter-db" ) - TSDB_DEFINE_SQL_TYPE(TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica") + TSDB_DEFINE_SQL_TYPE( TSDB_SQL_SYNC_DB_REPLICA, "sync db-replica") TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_MNODE, "create-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_MNODE, "drop-mnode" ) TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DNODE, "create-dnode" ) diff --git a/include/dnode/vnode/tq/tq.h b/include/dnode/vnode/tq/tq.h index 7993a8f1abdb2ad2f36964bfd861371ff96976dd..60a8c252c0b511e145e611814827626a48ed86bd 100644 --- a/include/dnode/vnode/tq/tq.h +++ b/include/dnode/vnode/tq/tq.h @@ -16,86 +16,76 @@ #ifndef _TD_TQ_H_ #define _TD_TQ_H_ +#include "common.h" #include "mallocator.h" #include "os.h" +#include "taoserror.h" +#include "taosmsg.h" +#include "tlist.h" #include "tutil.h" #ifdef __cplusplus extern "C" { #endif -typedef struct TmqMsgHead { +typedef struct STqMsgHead { int32_t protoVer; int32_t msgType; int64_t cgId; int64_t clientId; -} TmqMsgHead; +} STqMsgHead; -typedef struct TmqOneAck { +typedef struct STqOneAck { int64_t topicId; int64_t consumeOffset; -} TmqOneAck; +} STqOneAck; -typedef struct TmqAcks { +typedef struct STqAcks { int32_t ackNum; // should be sorted - TmqOneAck acks[]; -} TmqAcks; + STqOneAck acks[]; +} STqAcks; -// TODO: put msgs into common -typedef struct TmqConnectReq { - TmqMsgHead head; - TmqAcks acks; -} TmqConnectReq; - -typedef struct TmqConnectRsp { - TmqMsgHead head; - int8_t status; -} TmqConnectRsp; - -typedef struct TmqDisconnectReq { - TmqMsgHead head; -} TmqDiscconectReq; - -typedef struct TmqDisconnectRsp { - TmqMsgHead head; - int8_t status; -} TmqDisconnectRsp; +typedef struct STqSetCurReq { + STqMsgHead head; + int64_t topicId; + int64_t offset; +} STqSetCurReq; typedef struct STqConsumeReq { - TmqMsgHead head; - TmqAcks acks; + STqMsgHead head; + STqAcks acks; } STqConsumeReq; -typedef struct TmqMsgContent { +typedef struct STqMsgContent { int64_t topicId; int64_t msgLen; char msg[]; -} TmqMsgContent; +} STqMsgContent; typedef struct STqConsumeRsp { - TmqMsgHead head; + STqMsgHead head; int64_t bodySize; - TmqMsgContent msgs[]; + STqMsgContent msgs[]; } STqConsumeRsp; -typedef struct TmqSubscribeReq { - TmqMsgHead head; +typedef struct STqSubscribeReq { + STqMsgHead head; int32_t topicNum; int64_t topic[]; -} TmqSubscribeReq; +} STqSubscribeReq; -typedef struct tmqSubscribeRsp { - TmqMsgHead head; +typedef struct STqSubscribeRsp { + STqMsgHead head; int64_t vgId; char ep[TSDB_EP_LEN]; // TSDB_EP_LEN -} TmqSubscribeRsp; +} STqSubscribeRsp; -typedef struct TmqHeartbeatReq { -} TmqHeartbeatReq; +typedef struct STqHeartbeatReq { +} STqHeartbeatReq; -typedef struct TmqHeartbeatRsp { -} TmqHeartbeatRsp; +typedef struct STqHeartbeatRsp { +} STqHeartbeatRsp; typedef struct STqTopicVhandle { int64_t topicId; @@ -108,48 +98,54 @@ typedef struct STqTopicVhandle { #define TQ_BUFFER_SIZE 8 +typedef struct STqExec { + void* runtimeEnv; + SSDataBlock* (*exec)(void* runtimeEnv); + void* (*assign)(void* runtimeEnv, SSubmitBlk* inputData); + void (*clear)(void* runtimeEnv); + char* (*serialize)(struct STqExec*); + struct STqExec* (*deserialize)(char*); +} STqExec; + typedef struct STqBufferItem { int64_t offset; // executors are identical but not concurrent // so there must be a copy in each item - void* executor; - int64_t size; - void* content; -} STqBufferItem; + STqExec* executor; + int32_t status; + int64_t size; + void* content; +} STqMsgItem; -typedef struct STqBufferHandle { +typedef struct STqTopic { // char* topic; //c style, end with '\0' // int64_t cgId; // void* ahandle; - int64_t nextConsumeOffset; - int64_t floatingCursor; - int64_t topicId; - int32_t head; - int32_t tail; - STqBufferItem buffer[TQ_BUFFER_SIZE]; -} STqBufferHandle; + int64_t nextConsumeOffset; + int64_t floatingCursor; + int64_t topicId; + int32_t head; + int32_t tail; + STqMsgItem buffer[TQ_BUFFER_SIZE]; +} STqTopic; typedef struct STqListHandle { - STqBufferHandle bufHandle; + STqTopic topic; struct STqListHandle* next; -} STqListHandle; - -typedef struct STqGroupHandle { - int64_t cId; - int64_t cgId; - void* ahandle; - int32_t topicNum; - STqListHandle* head; -} STqGroupHandle; - -typedef struct STqQueryExec { - void* src; - STqBufferItem* dest; - void* executor; -} STqQueryExec; +} STqList; + +typedef struct STqGroup { + int64_t clientId; + int64_t cgId; + void* ahandle; + int32_t topicNum; + STqList* head; + SList* topicList; // SList + void* returnMsg; // SVReadMsg +} STqGroup; typedef struct STqQueryMsg { - STqQueryExec* exec; + STqMsgItem* item; struct STqQueryMsg* next; } STqQueryMsg; @@ -209,15 +205,15 @@ typedef void (*FTqDelete)(void*); #define TQ_DUP_INTXN_REWRITE 0 #define TQ_DUP_INTXN_REJECT 2 -static inline bool TqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } +static inline bool tqUpdateAppend(int32_t tqConfigFlag) { return tqConfigFlag & TQ_UPDATE_APPEND; } -static inline bool TqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } +static inline bool tqDupIntxnReject(int32_t tqConfigFlag) { return tqConfigFlag & TQ_DUP_INTXN_REJECT; } static const int8_t TQ_CONST_DELETE = TQ_ACTION_CONST; #define TQ_DELETE_TOKEN (void*)&TQ_CONST_DELETE -typedef struct TqMetaHandle { +typedef struct STqMetaHandle { int64_t key; int64_t offset; int64_t serializedSize; @@ -225,23 +221,25 @@ typedef struct TqMetaHandle { void* valueInTxn; } STqMetaHandle; -typedef struct TqMetaList { - STqMetaHandle handle; - struct TqMetaList* next; - // struct TqMetaList* inTxnPrev; - // struct TqMetaList* inTxnNext; - struct TqMetaList* unpersistPrev; - struct TqMetaList* unpersistNext; +typedef struct STqMetaList { + STqMetaHandle handle; + struct STqMetaList* next; + // struct STqMetaList* inTxnPrev; + // struct STqMetaList* inTxnNext; + struct STqMetaList* unpersistPrev; + struct STqMetaList* unpersistNext; } STqMetaList; -typedef struct TqMetaStore { +typedef struct STqMetaStore { STqMetaList* bucket[TQ_BUCKET_SIZE]; // a table head STqMetaList* unpersistHead; + // TODO:temporaral use, to be replaced by unified tfile int fileFd; // TODO:temporaral use, to be replaced by unified tfile - int idxFd; + int idxFd; + char* dirPath; int32_t tqConfigFlag; FTqSerialize pSerializer; @@ -250,8 +248,8 @@ typedef struct TqMetaStore { } STqMetaStore; typedef struct STQ { - // the collection of group handle - // the handle of kvstore + // the collection of groups + // the handle of meta kvstore char* path; STqCfg* tqConfig; STqLogReader* tqLogReader; @@ -266,23 +264,25 @@ void tqClose(STQ*); // void* will be replace by a msg type int tqPushMsg(STQ*, void* msg, int64_t version); int tqCommit(STQ*); -int tqSetCursor(STQ*, void* msg); - int tqConsume(STQ*, STqConsumeReq*); -STqGroupHandle* tqGetGroupHandle(STQ*, int64_t cId); +int tqSetCursor(STQ*, STqSetCurReq* pMsg); +int tqBufferSetOffset(STqTopic*, int64_t offset); + +STqTopic* tqFindTopic(STqGroup*, int64_t topicId); + +STqGroup* tqGetGroup(STQ*, int64_t clientId); + +STqGroup* tqOpenGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); +int tqCloseGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); +int tqRegisterContext(STqGroup*, void* ahandle); +int tqSendLaunchQuery(STqMsgItem*, int64_t offset); -STqGroupHandle* tqOpenTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqCloseTCGroup(STQ*, int64_t topicId, int64_t cgId, int64_t cId); -int tqMoveOffsetToNext(STqGroupHandle*); -int tqResetOffset(STQ*, int64_t topicId, int64_t cgId, int64_t offset); -int tqRegisterContext(STqGroupHandle*, void* ahandle); -int tqLaunchQuery(STqGroupHandle*); -int tqSendLaunchQuery(STqGroupHandle*); +int tqSerializeGroup(const STqGroup*, STqSerializedHead**); -int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead); +const void* tqDeserializeGroup(const STqSerializedHead*, STqGroup**); -const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** gHandle); +static int tqQueryExecuting(int32_t status) { return status; } #ifdef __cplusplus } diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 8458ad9da38d009524f8eee25d902ca305c0cdb2..081fcd7f7d04785db19ca3ac5abd162595c364be 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -197,8 +197,8 @@ void *vnodeParseReq(void *buf, SVnodeReq *pReq, uint8_t type); // int32_t daysToKeep0; // int32_t daysToKeep1; // int32_t daysToKeep2; -// int32_t minRowsPerFileBlock; -// int32_t maxRowsPerFileBlock; +// int32_t minRows; +// int32_t maxRows; // int8_t precision; // time resolution // int8_t compression; // int8_t cacheLastRow; diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index c01b79d7ff7dfe3df4b614dfac742636bf3346a6..2cbf26f8772927f67c3a882465cd66414be20a93 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -50,13 +50,15 @@ typedef struct SQueryProfileSummary { uint64_t resultSize; // generated result size in Kb. } SQueryProfileSummary; +int32_t schedulerInit(SSchedulerCfg *cfg); + /** * Process the query job, generated according to the query physical plan. * This is a synchronized API, and is also thread-safety. - * @param pJob + * @param qnodeList Qnode address list, element is SEpAddr * @return */ -int32_t scheduleQueryJob(struct SCatalog *pCatalog, void *pRpc, const SEpSet* pMgmtEps, SQueryDag* pDag, void** pJob); +int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob); int32_t scheduleFetchRows(void *pJob, void **data); @@ -70,6 +72,8 @@ int32_t scheduleCancelJob(void *pJob); void scheduleFreeJob(void *pJob); +void schedulerDestroy(void); + #ifdef __cplusplus } #endif diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 75aab24f83822ea4dfc2ec497fe71d9cf97df185..95824df5ac9adea0a260012369df2dd0fe4fe73e 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -339,6 +339,20 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) //"Invalid msg length") #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) //"Invalid msg type") +// tq +#define TSDB_CODE_TQ_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0A00) //"Invalid configuration") +#define TSDB_CODE_TQ_INIT_FAILED TAOS_DEF_ERROR_CODE(0, 0x0A01) //"Tq init failed") +#define TSDB_CODE_TQ_NO_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x0A02) //"No diskspace for tq") +#define TSDB_CODE_TQ_NO_DISK_PERMISSIONS TAOS_DEF_ERROR_CODE(0, 0x0A03) //"No permission for disk files") +#define TSDB_CODE_TQ_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x0A04) //"Data file(s) corrupted") +#define TSDB_CODE_TQ_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0A05) //"Out of memory") +#define TSDB_CODE_TQ_FILE_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x0A06) //"File already exists") +#define TSDB_CODE_TQ_FAILED_TO_CREATE_DIR TAOS_DEF_ERROR_CODE(0, 0x0A07) //"Failed to create dir") +#define TSDB_CODE_TQ_META_NO_SUCH_KEY TAOS_DEF_ERROR_CODE(0, 0x0A08) //"Target key not found") +#define TSDB_CODE_TQ_META_KEY_NOT_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A09) //"Target key not in transaction") +#define TSDB_CODE_TQ_META_KEY_DUP_IN_TXN TAOS_DEF_ERROR_CODE(0, 0x0A0A) //"Target key duplicated in transaction") +#define TSDB_CODE_TQ_GROUP_NOT_SET TAOS_DEF_ERROR_CODE(0, 0x0A0B) //"Group of corresponding client is not set by mnode") + // wal #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal") #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted") diff --git a/include/util/tlog.h b/include/util/tlog.h index 5e6604598dd22ad7755e4af51921fa3ee80a6abe..5c91398cdcf952046406d55e15911109a4c66435 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -42,11 +42,11 @@ extern int32_t qDebugFlag; extern int32_t wDebugFlag; extern int32_t sDebugFlag; extern int32_t tsdbDebugFlag; +extern int32_t tqDebugFlag; extern int32_t cqDebugFlag; extern int32_t debugFlag; extern int32_t ctgDebugFlag; - #define DEBUG_FATAL 1U #define DEBUG_ERROR DEBUG_FATAL #define DEBUG_WARN 2U diff --git a/include/util/tthread.h b/include/util/tthread.h index 0ff267dd1fef8e2e52d5032d15441604110655c0..7a5fd1f4c877024f2e3b8ade79a4094c1ef91ebc 100644 --- a/include/util/tthread.h +++ b/include/util/tthread.h @@ -24,8 +24,8 @@ extern "C" { #include "tdef.h" // create new thread -pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param); -// destory thread +pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param); +// destory thread bool taosDestoryThread(pthread_t* pthread); // thread running return true bool taosThreadRunning(pthread_t* pthread); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b6200de824822170e2827b041b1c768c3e7a48c1..9532789ca0e14d7871545b0bb1c54747ddec3608 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -153,7 +153,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { void* output = NULL; int32_t outputLen = 0; code = qParseQuerySql(pRequest->sqlstr, sqlLen, pRequest->requestId, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE); - if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_CREATE_DB) { + if (type == TSDB_SQL_CREATE_USER || type == TSDB_SQL_SHOW || type == TSDB_SQL_DROP_USER || type == TSDB_SQL_DROP_ACCT || type == TSDB_SQL_CREATE_DB || type == TSDB_SQL_CREATE_ACCT) { pRequest->type = type; pRequest->body.param = output; pRequest->body.paramLen = outputLen; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 36fa013f27e0b31f7d0124a3f6e8feccd19d345a..f8fd69b89a45e0cfb697ddf4584dd602a9b6be9f 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -407,7 +407,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { } // single table query error need to be handled here. - if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAGS_VAL) && + if ((cmd == TSDB_SQL_SELECT || cmd == TSDB_SQL_UPDATE_TAG_VAL) && (((rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID)) || rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || rpcMsg->code == TSDB_CODE_APP_NOT_READY)) { @@ -3176,6 +3176,15 @@ int32_t doBuildMsgSupp(SRequestObj *pRequest, SRequestMsgBody* pMsgBody) { case TSDB_SQL_CREATE_USER: pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_USER; break; + case TSDB_SQL_DROP_USER: + pMsgBody->msgType = TSDB_MSG_TYPE_DROP_USER; + break; + case TSDB_SQL_CREATE_ACCT: + pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_ACCT; + break; + case TSDB_SQL_DROP_ACCT: + pMsgBody->msgType = TSDB_MSG_TYPE_DROP_ACCT; + break; case TSDB_SQL_CREATE_DB: { pMsgBody->msgType = TSDB_MSG_TYPE_CREATE_DB; @@ -3328,7 +3337,7 @@ void initMsgHandleFp() { tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; - tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAG_VAL] = tscBuildUpdateTagMsg; tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; @@ -3383,6 +3392,10 @@ void initMsgHandleFp() { handleRequestRspFp[TSDB_SQL_CONNECT] = processConnectRsp; buildRequestMsgFp[TSDB_SQL_CREATE_USER] = doBuildMsgSupp; + buildRequestMsgFp[TSDB_SQL_DROP_USER] = doBuildMsgSupp; + + buildRequestMsgFp[TSDB_SQL_CREATE_ACCT] = doBuildMsgSupp; + buildRequestMsgFp[TSDB_SQL_DROP_ACCT] = doBuildMsgSupp; buildRequestMsgFp[TSDB_SQL_SHOW] = doBuildMsgSupp; handleRequestRspFp[TSDB_SQL_SHOW] = processShowRsp; diff --git a/source/client/src/tscEnv.c b/source/client/src/tscEnv.c index 023bd6ebe977c956f201d45fb63da31066d33ee2..76c37ca2f1734e0e79dc025f9678df48c8fb8e24 100644 --- a/source/client/src/tscEnv.c +++ b/source/client/src/tscEnv.c @@ -189,7 +189,6 @@ static void doDestroyRequest(void* p) { tfree(pRequest->pInfo); if (pRequest->body.pResInfo != NULL) { - tfree(pRequest->body.pResInfo->pData); tfree(pRequest->body.pResInfo->pMsg); tfree(pRequest->body.pResInfo); } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 944247e88e57901d57bdaf18a1025c53b08ad107..1c8cd6a4b6da560f0deec532c008863b6e7eb786 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -40,13 +40,13 @@ TEST(testCase, driverInit_Test) { TEST(testCase, connect_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +// assert(pConn != NULL); taos_close(pConn); } TEST(testCase, create_user_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); - assert(pConn != NULL); +// assert(pConn != NULL); TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { @@ -57,23 +57,36 @@ TEST(testCase, create_user_Test) { taos_close(pConn); } -//TEST(testCase, drop_user_Test) { -// TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); -// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { -// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} +TEST(testCase, create_account_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); + assert(pConn != NULL); -TEST(testCase, show_user_Test) { + TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, drop_account_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_user_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "show users"); TAOS_ROW pRow = NULL; @@ -89,10 +102,23 @@ TEST(testCase, show_user_Test) { taos_close(pConn); } -TEST(testCase, show_db_Test) { +TEST(testCase, drop_user_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "drop user abc"); + if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { + printf("failed to create user, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + +TEST(testCase, show_db_Test) { + TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); + TAOS_RES* pRes = taos_query(pConn, "show databases"); TAOS_ROW pRow = NULL; @@ -112,7 +138,7 @@ TEST(testCase, create_db_Test) { TAOS* pConn = taos_connect("ubuntu", "root", "taosdata", NULL, 0); assert(pConn != NULL); - TAOS_RES* pRes = taos_query(pConn, "create database abc"); + TAOS_RES* pRes = taos_query(pConn, "create database abc1"); TAOS_FIELD* pFields = taos_fetch_fields(pRes); ASSERT_TRUE(pFields == NULL); diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index b9a1e4cecbfe333591844c3ebbaebe5a96a0c741..083935c706e06abee8d07a74e9b4515350671e39 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -31,7 +31,7 @@ static struct { } global = {0}; void dmnSigintHandle(int signum, void *info, void *ctx) { - uError("singal:%d is received", signum); + uInfo("singal:%d is received", signum); global.stop = true; } diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 3402d3ff68f398329109d73813d8538b2347f1f0..50b1a1cf20fa2db10c401911aff6b8178f374d64 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -44,7 +44,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONSUME] = dndProcessVnodeQueryMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_CONNECT] = dndProcessVnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_MQ_DISCONNECT] = dndProcessVnodeWriteMsg; - pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET] = dndProcessVnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_MQ_SET_CUR] = dndProcessVnodeWriteMsg; // msg from client to mnode pMgmt->msgFp[TSDB_MSG_TYPE_CONNECT] = dndProcessMnodeReadMsg; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index d3f1b06a4a88fd5d96f760ed1ae7e2f9a968a3bf..cc4fc5b7d7e81abdbefb4ffcaabcda52b226ee44 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -17,11 +17,23 @@ #include "dndVnodes.h" #include "dndTransport.h" +typedef struct { + int32_t vgId; + int32_t vgVersion; + int8_t dropped; + uint64_t dbUid; + char db[TSDB_FULL_DB_NAME_LEN]; + char path[PATH_MAX + 20]; +} SWrapperCfg; + typedef struct { int32_t vgId; int32_t refCount; + int32_t vgVersion; int8_t dropped; int8_t accessState; + uint64_t dbUid; + char *db; char *path; SVnode *pImpl; taos_queue pWriteQ; @@ -32,13 +44,13 @@ typedef struct { } SVnodeObj; typedef struct { - int32_t vnodeNum; - int32_t opened; - int32_t failed; - int32_t threadIndex; - pthread_t *pThreadId; - SVnodeObj *pVnodes; - SDnode * pDnode; + int32_t vnodeNum; + int32_t opened; + int32_t failed; + int32_t threadIndex; + pthread_t thread; + SDnode *pDnode; + SWrapperCfg *pCfgs; } SVnodeThread; static int32_t dndInitVnodeReadWorker(SDnode *pDnode); @@ -73,16 +85,14 @@ void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEp void dndProcessVnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SVnodeMsg *pMsg); -static SVnodeObj * dndAcquireVnode(SDnode *pDnode, int32_t vgId); +static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId); static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode); -static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl); -static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode); +static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl); +static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode); static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes); -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes); +static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); static int32_t dndWriteVnodesToFile(SDnode *pDnode); -static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg); -static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode); static int32_t dndOpenVnodes(SDnode *pDnode); static void dndCloseVnodes(SDnode *pDnode); @@ -126,22 +136,25 @@ static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) { dTrace("vgId:%d, release vnode, refCount:%d", pVnode->vgId, refCount); } -static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, SVnode *pImpl) { +static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj * pVnode = calloc(1, sizeof(SVnodeObj)); + SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); if (pVnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pVnode->vgId = vgId; - pVnode->refCount = 0; + pVnode->vgId = pCfg->vgId; + pVnode->refCount = 1; pVnode->dropped = 0; pVnode->accessState = TSDB_VN_ALL_ACCCESS; pVnode->pImpl = pImpl; + pVnode->vgVersion = pCfg->vgVersion; + pVnode->dbUid = pCfg->dbUid; + pVnode->db = tstrdup(pCfg->db); + pVnode->path = tstrdup(pCfg->path); - pVnode->path = tstrdup(path); - if (pVnode->path == NULL) { + if (pVnode->path == NULL || pVnode->db == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -167,7 +180,7 @@ static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, S } taosWLockLatch(&pMgmt->latch); - int32_t code = taosHashPut(pMgmt->hash, &vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); + int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); taosWUnLockLatch(&pMgmt->latch); if (code != 0) { @@ -176,7 +189,7 @@ static int32_t dndCreateVnodeWrapper(SDnode *pDnode, int32_t vgId, char *path, S return code; } -static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) { +static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; taosWLockLatch(&pMgmt->latch); taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); @@ -195,6 +208,9 @@ static void dndDropVnodeWrapper(SDnode *pDnode, SVnodeObj *pVnode) { dndFreeVnodeWriteQueue(pDnode, pVnode); dndFreeVnodeApplyQueue(pDnode, pVnode); dndFreeVnodeSyncQueue(pDnode, pVnode); + free(pVnode->path); + free(pVnode->db); + free(pVnode); } static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { @@ -208,16 +224,16 @@ static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { void *pIter = taosHashIterate(pMgmt->hash, NULL); while (pIter) { SVnodeObj **ppVnode = pIter; - SVnodeObj * pVnode = *ppVnode; - if (pVnode) { + SVnodeObj *pVnode = *ppVnode; + if (pVnode && num < size) { + int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); + dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); + pVnodes[num] = (*ppVnode); num++; - if (num < size) { - int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); - pVnodes[num] = (*ppVnode); - } + pIter = taosHashIterate(pMgmt->hash, pIter); + } else { + taosHashCancelIterate(pMgmt->hash, pIter); } - pIter = taosHashIterate(pMgmt->hash, pIter); } taosRUnLockLatch(&pMgmt->latch); @@ -226,15 +242,15 @@ static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { return pVnodes; } -static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_t *numOfVnodes) { - int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; - int32_t len = 0; - int32_t maxLen = 30000; - char * content = calloc(1, maxLen + 1); - cJSON * root = NULL; - FILE * fp = NULL; - char file[PATH_MAX + 20] = {0}; - SVnodeObj *pVnodes = NULL; +static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_t *numOfVnodes) { + int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; + int32_t len = 0; + int32_t maxLen = 30000; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + FILE *fp = NULL; + char file[PATH_MAX + 20] = {0}; + SWrapperCfg *pCfgs = NULL; snprintf(file, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); @@ -270,31 +286,55 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SVnodeObj **ppVnodes, int32_ goto PRASE_VNODE_OVER; } - pVnodes = calloc(vnodesNum, sizeof(SVnodeObj)); - if (pVnodes == NULL) { + pCfgs = calloc(vnodesNum, sizeof(SWrapperCfg)); + if (pCfgs == NULL) { dError("failed to read %s since out of memory", file); goto PRASE_VNODE_OVER; } for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON * vnode = cJSON_GetArrayItem(vnodes, i); - SVnodeObj *pVnode = &pVnodes[i]; + cJSON *vnode = cJSON_GetArrayItem(vnodes, i); + SWrapperCfg *pCfg = &pCfgs[i]; cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); - if (!vgId || vgId->type != cJSON_String) { + if (!vgId || vgId->type != cJSON_Number) { dError("failed to read %s since vgId not found", file); goto PRASE_VNODE_OVER; } - pVnode->vgId = atoi(vgId->valuestring); + pCfg->vgId = vgId->valueint; + snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCfg->vgId); cJSON *dropped = cJSON_GetObjectItem(vnode, "dropped"); - if (!dropped || dropped->type != cJSON_String) { + if (!dropped || dropped->type != cJSON_Number) { dError("failed to read %s since dropped not found", file); goto PRASE_VNODE_OVER; } - pVnode->dropped = atoi(vnode->valuestring); + pCfg->dropped = dropped->valueint; + + cJSON *vgVersion = cJSON_GetObjectItem(vnode, "vgVersion"); + if (!vgVersion || vgVersion->type != cJSON_Number) { + dError("failed to read %s since vgVersion not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->vgVersion = vgVersion->valueint; + + cJSON *dbUid = cJSON_GetObjectItem(vnode, "dbUid"); + if (!dbUid || dbUid->type != cJSON_String) { + dError("failed to read %s since dbUid not found", file); + goto PRASE_VNODE_OVER; + } + pCfg->dbUid = atoll(dbUid->valuestring); + + cJSON *db = cJSON_GetObjectItem(vnode, "db"); + if (!db || db->type != cJSON_String) { + dError("failed to read %s since db not found", file); + goto PRASE_VNODE_OVER; + } + tstrncpy(pCfg->db, db->valuestring, TSDB_FULL_DB_NAME_LEN); } + *ppCfgs = pCfgs; + *numOfVnodes = vnodesNum; code = 0; dInfo("succcessed to read file %s", file); @@ -313,30 +353,35 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { snprintf(realfile, PATH_MAX + 20, "%s/vnodes.json", pDnode->dir.vnodes); FILE *fp = fopen(file, "w"); - if (fp != NULL) { + if (fp == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); dError("failed to write %s since %s", file, terrstr()); return -1; } - - int32_t len = 0; - int32_t maxLen = 30000; - char * content = calloc(1, maxLen + 1); int32_t numOfVnodes = 0; SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); + int32_t len = 0; + int32_t maxLen = 65536; + char *content = calloc(1, maxLen + 1); + len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"vnodes\": [{\n"); + len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); for (int32_t i = 0; i < numOfVnodes; ++i) { SVnodeObj *pVnode = pVnodes[i]; - len += snprintf(content + len, maxLen - len, " \"vgId\": \"%d\",\n", pVnode->vgId); - len += snprintf(content + len, maxLen - len, " \"dropped\": \"%d\"\n", pVnode->dropped); + len += snprintf(content + len, maxLen - len, " {\n"); + len += snprintf(content + len, maxLen - len, " \"vgId\": %d,\n", pVnode->vgId); + len += snprintf(content + len, maxLen - len, " \"dropped\": %d,\n", pVnode->dropped); + len += snprintf(content + len, maxLen - len, " \"vgVersion\": %d,\n", pVnode->vgVersion); + len += snprintf(content + len, maxLen - len, " \"dbUid\": \"%" PRIu64 "\",\n", pVnode->dbUid); + len += snprintf(content + len, maxLen - len, " \"db\": \"%s\"\n", pVnode->db); if (i < numOfVnodes - 1) { - len += snprintf(content + len, maxLen - len, " },{\n"); + len += snprintf(content + len, maxLen - len, " },\n"); } else { - len += snprintf(content + len, maxLen - len, " }]\n"); + len += snprintf(content + len, maxLen - len, " }\n"); } } + len += snprintf(content + len, maxLen - len, " ]\n"); len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); @@ -358,74 +403,29 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { return taosRenameFile(file, realfile); } -static int32_t dndCreateVnode(SDnode *pDnode, int32_t vgId, SVnodeCfg *pCfg) { - char path[PATH_MAX + 20] = {0}; - snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, vgId); - // SVnode *pImpl = vnodeCreate(vgId, path, pCfg); - - SVnode *pImpl = vnodeOpen(path, NULL); - if (pImpl == NULL) { - return -1; - } - - int32_t code = dndCreateVnodeWrapper(pDnode, vgId, path, pImpl); - if (code != 0) { - vnodeClose(pImpl); - vnodeDestroy(path); - terrno = code; - return code; - } - - code = dndWriteVnodesToFile(pDnode); - if (code != 0) { - vnodeClose(pImpl); - vnodeDestroy(path); - terrno = code; - return code; - } - - return 0; -} - -static int32_t dndDropVnode(SDnode *pDnode, SVnodeObj *pVnode) { - pVnode->dropped = 1; - if (dndWriteVnodesToFile(pDnode) != 0) { - pVnode->dropped = 0; - return -1; - } - - dndDropVnodeWrapper(pDnode, pVnode); - vnodeClose(pVnode->pImpl); - vnodeDestroy(pVnode->path); - dndWriteVnodesToFile(pDnode); - return 0; -} - static void *dnodeOpenVnodeFunc(void *param) { SVnodeThread *pThread = param; - SDnode * pDnode = pThread->pDnode; - SVnodesMgmt * pMgmt = &pDnode->vmgmt; + SDnode *pDnode = pThread->pDnode; + SVnodesMgmt *pMgmt = &pDnode->vmgmt; dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); setThreadName("open-vnodes"); for (int32_t v = 0; v < pThread->vnodeNum; ++v) { - SVnodeObj *pVnode = &pThread->pVnodes[v]; + SWrapperCfg *pCfg = &pThread->pCfgs[v]; char stepDesc[TSDB_STEP_DESC_LEN] = {0}; - snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pVnode->vgId, + snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to restore, %d of %d have been opened", pCfg->vgId, pMgmt->openVnodes, pMgmt->totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - char path[PATH_MAX + 20] = {0}; - snprintf(path, sizeof(path), "%s/vnode%d", pDnode->dir.vnodes, pVnode->vgId); - SVnode *pImpl = vnodeOpen(path, NULL); + SVnode *pImpl = vnodeOpen(pCfg->path, NULL); if (pImpl == NULL) { - dError("vgId:%d, failed to open vnode by thread:%d", pVnode->vgId, pThread->threadIndex); + dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->failed++; } else { - dndCreateVnodeWrapper(pDnode, pVnode->vgId, path, pImpl); - dDebug("vgId:%d, is opened by thread:%d", pVnode->vgId, pThread->threadIndex); + dndOpenVnode(pDnode, pCfg, pImpl); + dDebug("vgId:%d, is opened by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->opened++; } @@ -448,9 +448,9 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { return -1; } - SVnodeObj *pVnodes = NULL; - int32_t numOfVnodes = 0; - if (dndGetVnodesFromFile(pDnode, &pVnodes, &numOfVnodes) != 0) { + SWrapperCfg *pCfgs = NULL; + int32_t numOfVnodes = 0; + if (dndGetVnodesFromFile(pDnode, &pCfgs, &numOfVnodes) != 0) { dInfo("failed to get vnode list from disk since %s", terrstr()); return -1; } @@ -463,13 +463,14 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); for (int32_t t = 0; t < threadNum; ++t) { threads[t].threadIndex = t; - threads[t].pVnodes = calloc(vnodesPerThread, sizeof(SVnodeObj)); + threads[t].pDnode = pDnode; + threads[t].pCfgs = calloc(vnodesPerThread, sizeof(SWrapperCfg)); } for (int32_t v = 0; v < numOfVnodes; ++v) { int32_t t = v % threadNum; SVnodeThread *pThread = &threads[t]; - pThread->pVnodes[pThread->vnodeNum++] = pVnodes[v]; + pThread->pCfgs[pThread->vnodeNum++] = pCfgs[v]; } dInfo("start %d threads to open %d vnodes", threadNum, numOfVnodes); @@ -478,19 +479,25 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { SVnodeThread *pThread = &threads[t]; if (pThread->vnodeNum == 0) continue; - pThread->pThreadId = taosCreateThread(dnodeOpenVnodeFunc, pThread); - if (pThread->pThreadId == NULL) { + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + if (pthread_create(&pThread->thread, &thAttr, dnodeOpenVnodeFunc, pThread) != 0) { dError("thread:%d, failed to create thread to open vnode, reason:%s", pThread->threadIndex, strerror(errno)); } + + pthread_attr_destroy(&thAttr); } for (int32_t t = 0; t < threadNum; ++t) { SVnodeThread *pThread = &threads[t]; - taosDestoryThread(pThread->pThreadId); - pThread->pThreadId = NULL; - free(pThread->pVnodes); + if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) { + pthread_join(pThread->thread, NULL); + } + free(pThread->pCfgs); } free(threads); + free(pCfgs); if (pMgmt->openVnodes != pMgmt->totalVnodes) { dError("there are total vnodes:%d, opened:%d", pMgmt->totalVnodes, pMgmt->openVnodes); @@ -508,7 +515,8 @@ static void dndCloseVnodes(SDnode *pDnode) { SVnodeObj **pVnodes = dndGetVnodesFromHash(pDnode, &numOfVnodes); for (int32_t i = 0; i < numOfVnodes; ++i) { - dndDropVnodeWrapper(pDnode, pVnodes[i]); + dndReleaseVnode(pDnode, pVnodes[i]); + dndCloseVnode(pDnode, pVnodes[i]); } if (pVnodes != NULL) { @@ -523,11 +531,12 @@ static void dndCloseVnodes(SDnode *pDnode) { dInfo("total vnodes:%d are all closed", numOfVnodes); } -static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg *pCfg) { +static SCreateVnodeMsg *dndParseCreateVnodeReq(SRpcMsg *rpcMsg) { SCreateVnodeMsg *pCreate = rpcMsg->pCont; pCreate->vgId = htonl(pCreate->vgId); pCreate->dnodeId = htonl(pCreate->dnodeId); pCreate->dbUid = htobe64(pCreate->dbUid); + pCreate->vgVersion = htonl(pCreate->vgVersion); pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); pCreate->totalBlocks = htonl(pCreate->totalBlocks); pCreate->daysPerFile = htonl(pCreate->daysPerFile); @@ -544,9 +553,10 @@ static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg pReplica->port = htons(pReplica->port); } - *vgId = pCreate->vgId; + return pCreate; +} -#if 0 +static void dndGenerateVnodeCfg(SCreateVnodeMsg *pCreate, SVnodeCfg *pCfg) { pCfg->wsize = pCreate->cacheBlockSize; pCfg->ssize = pCreate->cacheBlockSize; pCfg->wsize = pCreate->cacheBlockSize; @@ -567,8 +577,15 @@ static int32_t dndParseCreateVnodeReq(SRpcMsg *rpcMsg, int32_t *vgId, SVnodeCfg pCfg->walCfg.rollPeriod = 128; pCfg->walCfg.segSize = 128; pCfg->walCfg.vgId = pCreate->vgId; -#endif - return 0; +} + +static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeMsg *pCreate, SWrapperCfg *pCfg) { + memcpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN); + pCfg->dbUid = pCreate->dbUid; + pCfg->dropped = 0; + snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCreate->vgId); + pCfg->vgId = pCreate->vgId; + pCfg->vgVersion = pCreate->vgVersion; } static SDropVnodeMsg *vnodeParseDropVnodeReq(SRpcMsg *rpcMsg) { @@ -584,48 +601,83 @@ static SAuthVnodeMsg *vnodeParseAuthVnodeReq(SRpcMsg *rpcMsg) { } static int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SCreateVnodeMsg *pCreate = dndParseCreateVnodeReq(rpcMsg); + dDebug("vgId:%d, create vnode req is received", pCreate->vgId); + SVnodeCfg vnodeCfg = {0}; - int32_t vgId = 0; + dndGenerateVnodeCfg(pCreate, &vnodeCfg); - dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); - dDebug("vgId:%d, create vnode req is received", vgId); + SWrapperCfg wrapperCfg = {0}; + dndGenerateWrapperCfg(pDnode, pCreate, &wrapperCfg); - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + SVnodeObj *pVnode = dndAcquireVnode(pDnode, pCreate->vgId); if (pVnode != NULL) { - dDebug("vgId:%d, already exist, return success", vgId); + dDebug("vgId:%d, already exist, return success", pCreate->vgId); dndReleaseVnode(pDnode, pVnode); return 0; } - if (dndCreateVnode(pDnode, vgId, &vnodeCfg) != 0) { - dError("vgId:%d, failed to create vnode since %s", vgId, terrstr()); - return terrno; + SVnode *pImpl = vnodeOpen(wrapperCfg.path, NULL /*pCfg*/); + if (pImpl == NULL) { + return -1; + } + + int32_t code = dndOpenVnode(pDnode, &wrapperCfg, pImpl); + if (code != 0) { + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; + } + + code = dndWriteVnodesToFile(pDnode); + if (code != 0) { + vnodeClose(pImpl); + vnodeDestroy(wrapperCfg.path); + terrno = code; + return code; } return 0; } static int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { + SAlterVnodeMsg *pAlter = (SAlterVnodeMsg *)dndParseCreateVnodeReq(rpcMsg); + dDebug("vgId:%d, alter vnode req is received", pAlter->vgId); + SVnodeCfg vnodeCfg = {0}; - int32_t vgId = 0; + dndGenerateVnodeCfg(pAlter, &vnodeCfg); - dndParseCreateVnodeReq(rpcMsg, &vgId, &vnodeCfg); - dDebug("vgId:%d, alter vnode req is received", vgId); + SWrapperCfg wrapperCfg = {0}; + dndGenerateWrapperCfg(pDnode, pAlter, &wrapperCfg); - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); + SVnodeObj *pVnode = dndAcquireVnode(pDnode, pAlter->vgId); if (pVnode == NULL) { - dDebug("vgId:%d, failed to alter vnode since %s", vgId, terrstr()); + dDebug("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); return terrno; } + if (wrapperCfg.vgVersion == pVnode->vgVersion) { + dndReleaseVnode(pDnode, pVnode); + dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", pAlter->vgId); + return 0; + } + if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { - dError("vgId:%d, failed to alter vnode since %s", vgId, terrstr()); + dError("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); dndReleaseVnode(pDnode, pVnode); return terrno; } + int32_t oldVersion = pVnode->vgVersion; + pVnode->vgVersion = wrapperCfg.vgVersion; + int32_t code = dndWriteVnodesToFile(pDnode); + if (code != 0) { + pVnode->vgVersion = oldVersion; + } + dndReleaseVnode(pDnode, pVnode); - return 0; + return code; } static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { @@ -637,15 +689,21 @@ static int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *rpcMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) { dDebug("vgId:%d, failed to drop since %s", vgId, terrstr()); - return terrno; + return 0; } - if (dndDropVnode(pDnode, pVnode) != 0) { - dError("vgId:%d, failed to drop vnode since %s", vgId, terrstr()); - dndReleaseVnode(pDnode, pVnode); + pVnode->dropped = 1; + if (dndWriteVnodesToFile(pDnode) != 0) { + pVnode->dropped = 0; return terrno; } + dndReleaseVnode(pDnode, pVnode); + dndCloseVnode(pDnode, pVnode); + vnodeClose(pVnode->pImpl); + vnodeDestroy(pVnode->path); + dndWriteVnodesToFile(pDnode); + return 0; } @@ -738,12 +796,10 @@ static void dndProcessVnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { break; } - if (code != 0) { - SRpcMsg rsp = {.code = code, .handle = pMsg->handle}; - rpcSendResponse(&rsp); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } + SRpcMsg rsp = {.code = code, .handle = pMsg->handle, .ahandle = pMsg->ahandle}; + rpcSendResponse(&rsp); + rpcFreeCont(pMsg->pCont); + taosFreeQitem(pMsg); } static void dndProcessVnodeQueryQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) { @@ -756,7 +812,7 @@ static void dndProcessVnodeFetchQueue(SVnodeObj *pVnode, SVnodeMsg *pMsg) { static void dndProcessVnodeWriteQueue(SVnodeObj *pVnode, taos_qall qall, int32_t numOfMsgs) { SVnodeMsg *pMsg = vnodeInitMsg(numOfMsgs); - SRpcMsg * pRpcMsg = NULL; + SRpcMsg *pRpcMsg = NULL; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(qall, (void **)&pRpcMsg); @@ -1029,7 +1085,7 @@ static void dndFreeVnodeApplyQueue(SDnode *pDnode, SVnodeObj *pVnode) { } static int32_t dndInitVnodeWriteWorker(SDnode *pDnode) { - SVnodesMgmt * pMgmt = &pDnode->vmgmt; + SVnodesMgmt *pMgmt = &pDnode->vmgmt; SMWorkerPool *pPool = &pMgmt->writePool; pPool->name = "vnode-write"; pPool->max = pDnode->opt.numOfCores; @@ -1137,12 +1193,12 @@ void dndGetVnodeLoads(SDnode *pDnode, SVnodeLoads *pLoads) { pLoads->num = taosHashGetSize(pMgmt->hash); int32_t v = 0; - void * pIter = taosHashIterate(pMgmt->hash, NULL); + void *pIter = taosHashIterate(pMgmt->hash, NULL); while (pIter) { SVnodeObj **ppVnode = pIter; if (ppVnode == NULL || *ppVnode == NULL) continue; - SVnodeObj * pVnode = *ppVnode; + SVnodeObj *pVnode = *ppVnode; SVnodeLoad *pLoad = &pLoads->data[v++]; vnodeGetLoad(pVnode->pImpl, pLoad); diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index 0dbee0d3379daa55e69ba6ed5e0b2beaa400dec7..a4996ecb3bfc0792add96700c51c4b83dcf6d80e 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -176,6 +176,12 @@ SDnode *dndInit(SDnodeOpt *pOption) { return NULL; } + if (vnodeInit(1) != 0) { + dError("failed to init vnode env"); + dndCleanup(pDnode); + return NULL; + } + if (dndInitDnode(pDnode) != 0) { dError("failed to init dnode"); dndCleanup(pDnode); @@ -222,8 +228,10 @@ void dndCleanup(SDnode *pDnode) { dndCleanupMnode(pDnode); dndCleanupVnodes(pDnode); dndCleanupDnode(pDnode); + vnodeClear(); walCleanUp(); rpcCleanup(); + dndCleanupEnv(pDnode); free(pDnode); dInfo("TDengine is cleaned up successfully"); diff --git a/source/dnode/mgmt/impl/test/db/db.cpp b/source/dnode/mgmt/impl/test/db/db.cpp index d465a62f2d5269632c4f5022e0bafca3a1e45670..204afa111fdd509aac44937774410a8165905314 100644 --- a/source/dnode/mgmt/impl/test/db/db.cpp +++ b/source/dnode/mgmt/impl/test/db/db.cpp @@ -210,8 +210,8 @@ TEST_F(DndTestDb, 02_Create_Alter_Drop_Db) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; @@ -375,8 +375,8 @@ TEST_F(DndTestDb, 03_Create_Use_Restart_Use_Db) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; diff --git a/source/dnode/mgmt/impl/test/stb/stb.cpp b/source/dnode/mgmt/impl/test/stb/stb.cpp index ee8c86bcea3b5764ca3158deabbced027b70d4ee..c12e8eadf45111af660126837fa1157212f8bd57 100644 --- a/source/dnode/mgmt/impl/test/stb/stb.cpp +++ b/source/dnode/mgmt/impl/test/stb/stb.cpp @@ -187,8 +187,8 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; diff --git a/source/dnode/mgmt/impl/test/sut/deploy.cpp b/source/dnode/mgmt/impl/test/sut/deploy.cpp index f2010b581388f24359ff2c15f4517a3fded22047..7cf469f8e253f0354b74cbdda7d2e8cf6cc8a2c5 100644 --- a/source/dnode/mgmt/impl/test/sut/deploy.cpp +++ b/source/dnode/mgmt/impl/test/sut/deploy.cpp @@ -16,7 +16,7 @@ #include "deploy.h" void initLog(const char* path) { - dDebugFlag = 143; + dDebugFlag = 207; vDebugFlag = 0; mDebugFlag = 207; cDebugFlag = 0; diff --git a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp index 3f16cd87d889902b3e7d34fbbf6a2cb2d83e6771..9149be9cd838be6ddd708a1fa56e85f6dc5f2dae 100644 --- a/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp +++ b/source/dnode/mgmt/impl/test/vgroup/vgroup.cpp @@ -176,49 +176,112 @@ SServer* DndTestVgroup::pServer; SClient* DndTestVgroup::pClient; int32_t DndTestVgroup::connId; - TEST_F(DndTestVgroup, 01_Create_Restart_Drop_Vnode) { { - SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg)); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->minRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replica = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->selfIndex = 0; - for (int r = 0; r < pReq->replica; ++r) { - SReplica* pReplica = &pReq->replicas[r]; - pReplica->id = htonl(1); - pReplica->port = htons(9150); + for (int i = 0; i < 3; ++i) { + SCreateVnodeMsg* pReq = (SCreateVnodeMsg*)rpcMallocCont(sizeof(SCreateVnodeMsg)); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + pReq->vgVersion = htonl(1); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->minRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replica = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->selfIndex = 0; + for (int r = 0; r < pReq->replica; ++r) { + SReplica* pReplica = &pReq->replicas[r]; + pReplica->id = htonl(1); + pReplica->port = htons(9150); + } + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SCreateVnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); } + } - SRpcMsg rpcMsg = {0}; - rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateVnodeMsg); - rpcMsg.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN; - - sendMsg(pClient, &rpcMsg); - SRpcMsg* pMsg = pClient->pRsp; - ASSERT_NE(pMsg, nullptr); - ASSERT_EQ(pMsg->code, 0); - taosMsleep(1000000); + { + for (int i = 0; i < 3; ++i) { + SAlterVnodeMsg* pReq = (SAlterVnodeMsg*)rpcMallocCont(sizeof(SAlterVnodeMsg)); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + pReq->vgVersion = htonl(2); + pReq->cacheBlockSize = htonl(16); + pReq->totalBlocks = htonl(10); + pReq->daysPerFile = htonl(10); + pReq->daysToKeep0 = htonl(3650); + pReq->daysToKeep1 = htonl(3650); + pReq->daysToKeep2 = htonl(3650); + pReq->minRows = htonl(100); + pReq->minRows = htonl(4096); + pReq->commitTime = htonl(3600); + pReq->fsyncPeriod = htonl(3000); + pReq->walLevel = 1; + pReq->precision = 0; + pReq->compression = 2; + pReq->replica = 1; + pReq->quorum = 1; + pReq->update = 0; + pReq->cacheLastRow = 0; + pReq->selfIndex = 0; + for (int r = 0; r < pReq->replica; ++r) { + SReplica* pReplica = &pReq->replicas[r]; + pReplica->id = htonl(1); + pReplica->port = htons(9150); + } + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SAlterVnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_ALTER_VNODE_IN; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } } + { + for (int i = 0; i < 3; ++i) { + SDropVnodeMsg* pReq = (SDropVnodeMsg*)rpcMallocCont(sizeof(SDropVnodeMsg)); + pReq->vgId = htonl(2); + pReq->dnodeId = htonl(1); + strcpy(pReq->db, "1.d1"); + pReq->dbUid = htobe64(9527); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SDropVnodeMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN; + + sendMsg(pClient, &rpcMsg); + SRpcMsg* pMsg = pClient->pRsp; + ASSERT_NE(pMsg, nullptr); + ASSERT_EQ(pMsg->code, 0); + } + } } - diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 36b6725737bb3b4cd6a070da92fe01060f80d17e..da5492057458b09aed332168919ce1b38a46e959 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -61,15 +61,14 @@ typedef enum { } EAuthOp; typedef enum { - TRN_STAGE_PREPARE = 1, - TRN_STAGE_EXECUTE = 2, + TRN_STAGE_PREPARE = 0, + TRN_STAGE_EXECUTE = 1, + TRN_STAGE_ROLLBACK = 2, TRN_STAGE_COMMIT = 3, - TRN_STAGE_ROLLBACK = 4, - TRN_STAGE_RETRY = 5, - TRN_STAGE_OVER = 6, + TRN_STAGE_OVER = 4, } ETrnStage; -typedef enum { TRN_POLICY_ROLLBACK = 1, TRN_POLICY_RETRY = 2 } ETrnPolicy; +typedef enum { TRN_POLICY_ROLLBACK = 0, TRN_POLICY_RETRY = 1 } ETrnPolicy; typedef enum { DND_STATUS_OFFLINE = 0, diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index e9913803bd7f2fe88db87c2ea2b44aafb062f554..ba72d9553721b9220aa520b8bb11b46c1b7baeb5 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -70,6 +70,7 @@ typedef struct SMnode { tmr_h timer; char *path; SMnodeCfg cfg; + int64_t checkTime; SSdb *pSdb; SDnode *pDnode; SArray *pSteps; diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 5c15e2f987d89ad00c26201bfcdc85352d2a71b0..a8d37ba655ef52c6f4fdf4b77938cab4bb0a6046 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -22,6 +22,16 @@ extern "C" { #endif +typedef struct { + SEpSet epSet; + int8_t msgType; + int8_t msgSent; + int8_t msgReceived; + int32_t errCode; + int32_t contLen; + void *pCont; +} STransAction; + int32_t mndInitTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode); @@ -30,12 +40,15 @@ void mndTransDrop(STrans *pTrans); int32_t mndTransAppendRedolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendUndolog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); -int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); -int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); +int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); +int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); + int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code); -char *mndTransStageStr(ETrnStage stage); -char *mndTransPolicyStr(ETrnPolicy policy); +void mndTransHandleActionRsp(SMnodeMsg *pMsg); + +char *mndTransStageStr(ETrnStage stage); +char *mndTransPolicyStr(ETrnPolicy policy); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 75b0acbd0a6df8e98b1a3304edb15701284805dc..426e8500f2424b55a1b836b5f91b57b2be7a02f2 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -17,7 +17,8 @@ #include "mndAcct.h" #include "mndShow.h" -#define SDB_ACCT_VER 1 +#define TSDB_ACCT_VER_NUMBER 1 +#define TSDB_ACCT_RESERVE_SIZE 64 static int32_t mndCreateDefaultAcct(SMnode *pMnode); static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct); @@ -70,7 +71,7 @@ static int32_t mndCreateDefaultAcct(SMnode *pMnode) { } static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, TSDB_ACCT_VER_NUMBER, sizeof(SAcctObj)); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -85,6 +86,7 @@ static SSdbRaw *mndAcctActionEncode(SAcctObj *pAcct) { SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.maxStreams) SDB_SET_INT64(pRaw, dataPos, pAcct->cfg.maxStorage) SDB_SET_INT32(pRaw, dataPos, pAcct->cfg.accessState) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_ACCT_RESERVE_SIZE) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -94,7 +96,7 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_ACCT_VER) { + if (sver != TSDB_ACCT_VER_NUMBER) { mError("failed to decode acct since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -116,13 +118,13 @@ static SSdbRow *mndAcctActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.maxStreams) SDB_GET_INT64(pRaw, pRow, dataPos, &pAcct->cfg.maxStorage) SDB_GET_INT32(pRaw, pRow, dataPos, &pAcct->cfg.accessState) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_ACCT_RESERVE_SIZE) return pRow; } static int32_t mndAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct) { mTrace("acct:%s, perform insert action", pAcct->acct); - memset(&pAcct->info, 0, sizeof(SAcctInfo)); return 0; } @@ -134,12 +136,9 @@ static int32_t mndAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct) { static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOldAcct, SAcctObj *pNewAcct) { mTrace("acct:%s, perform update action", pOldAcct->acct); - memcpy(pOldAcct->acct, pNewAcct->acct, TSDB_USER_LEN); - pOldAcct->createdTime = pNewAcct->createdTime; pOldAcct->updateTime = pNewAcct->updateTime; - pOldAcct->acctId = pNewAcct->acctId; pOldAcct->status = pNewAcct->status; - pOldAcct->cfg = pNewAcct->cfg; + memcpy(&pOldAcct->cfg, &pNewAcct->cfg, sizeof(SAcctInfo)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 7c53a4ebd0516c13856e9d36d506c8babfc880c8..7e9a12e8890037ce734a4b93a6a35dc9f5ef9cdc 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -18,7 +18,8 @@ #include "mndShow.h" #include "mndTrans.h" -#define SDB_CLUSTER_VER 1 +#define TSDB_CLUSTER_VER_NUMBE 1 +#define TSDB_CLUSTER_RESERVE_SIZE 64 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw); @@ -62,7 +63,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { } static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, SDB_CLUSTER_VER, sizeof(SClusterObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_CLUSTER, TSDB_CLUSTER_VER_NUMBE, sizeof(SClusterObj)); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -70,6 +71,7 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { SDB_SET_INT64(pRaw, dataPos, pCluster->createdTime) SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime) SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_CLUSTER_RESERVE_SIZE) return pRaw; } @@ -78,7 +80,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_CLUSTER_VER) { + if (sver != TSDB_CLUSTER_VER_NUMBE) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to decode cluster since %s", terrstr()); return NULL; @@ -93,6 +95,7 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pCluster->updateTime) SDB_GET_BINARY(pRaw, pRow, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_CLUSTER_RESERVE_SIZE) return pRow; } @@ -169,6 +172,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg pShow->numOfRows = 1; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + strcpy(pMeta->tbFname, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index e85edf66da85011fe9b1c32e18a3568fb3ecc477..108896c121eeb1d3d93ffd3a1a9c93cc6f413235 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -16,13 +16,12 @@ #define _DEFAULT_SOURCE #include "mndDb.h" #include "mndDnode.h" -#include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" -#define TSDB_DB_VER_NUM 1 +#define TSDB_DB_VER_NUMBER 1 #define TSDB_DB_RESERVE_SIZE 64 static SSdbRaw *mndDbActionEncode(SDbObj *pDb); @@ -66,7 +65,7 @@ int32_t mndInitDb(SMnode *pMnode) { void mndCleanupDb(SMnode *pMnode) {} static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUM, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -106,7 +105,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_DB_VER_NUM) { + if (sver != TSDB_DB_VER_NUMBER) { mError("failed to decode db since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -160,7 +159,7 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { mTrace("db:%s, perform update action", pOldDb->name); - pOldDb->updateTime = pNewDb->createdTime; + pOldDb->updateTime = pNewDb->updateTime; pOldDb->cfgVersion = pNewDb->cfgVersion; pOldDb->vgVersion = pNewDb->vgVersion; memcpy(&pOldDb->cfg, &pNewDb->cfg, sizeof(SDbCfg)); @@ -168,8 +167,12 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOldDb, SDbObj *pNewDb) { } SDbObj *mndAcquireDb(SMnode *pMnode, char *db) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_DB, db); + SSdb *pSdb = pMnode->pSdb; + SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + } + return pDb; } void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { @@ -242,68 +245,74 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; } -static int32_t mndSetRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); - if (pDbRaw == NULL || mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1; - sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING); + if (pDbRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pDbRaw) != 0) return -1; + if (sdbSetRawStatus(pDbRaw, SDB_STATUS_CREATING) != 0) return -1; - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); - if (pVgRaw == NULL || mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; - sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1; } return 0; } -static int32_t mndSetUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); - if (pDbRaw == NULL || mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1; - sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED); + if (pDbRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pDbRaw) != 0) return -1; + if (sdbSetRawStatus(pDbRaw, SDB_STATUS_DROPPED) != 0) return -1; - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); - if (pVgRaw == NULL || mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1; - sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED); + if (pVgRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_DROPPED) != 0) return -1; } return 0; } -static int32_t mndSetCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { +static int32_t mndSetCreateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { SSdbRaw *pDbRaw = mndDbActionEncode(pDb); - if (pDbRaw == NULL || mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; - sdbSetRawStatus(pDbRaw, SDB_STATUS_READY); + if (pDbRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pDbRaw) != 0) return -1; + if (sdbSetRawStatus(pDbRaw, SDB_STATUS_READY) != 0) return -1; - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { + for (int32_t v = 0; v < pDb->cfg.numOfVgroups; ++v) { SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroups + v); - if (pVgRaw == NULL || mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; - sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; } return 0; } -static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { - SVgObj *pVgroup = pVgroups + v; +static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + for (int32_t vg = 0; vg < pDb->cfg.numOfVgroups; ++vg) { + SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) { - return -1; - } + STransAction action = {0}; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SEpSet epset = mndGetDnodeEpset(pDnode); + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); SCreateVnodeMsg *pMsg = mndBuildCreateVnodeMsg(pMnode, pDnode, pDb, pVgroup); - if (pMsg == NULL) { - return -1; - } + if (pMsg == NULL) return -1; - if (mndTransAppendRedoAction(pTrans, &epset, TSDB_MSG_TYPE_ALTER_VNODE_IN, sizeof(SCreateVnodeMsg), pMsg) != 0) { + action.pCont = pMsg; + action.contLen = sizeof(SCreateVnodeMsg); + action.msgType = TSDB_MSG_TYPE_CREATE_VNODE_IN; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pMsg); return -1; } @@ -313,26 +322,26 @@ static int32_t mndSetRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV return 0; } -static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { - for (int v = 0; v < pDb->cfg.numOfVgroups; ++v) { - SVgObj *pVgroup = pVgroups + v; +static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { + for (int32_t vg = 0; vg < pDb->cfg.numOfVgroups; ++vg) { + SVgObj *pVgroup = pVgroups + vg; for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { - SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pDnode == NULL) { - return -1; - } + STransAction action = {0}; + SVnodeGid *pVgid = pVgroup->vnodeGid + vn; - SEpSet epset = mndGetDnodeEpset(pDnode); + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pDnode == NULL) return -1; + action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); SDropVnodeMsg *pMsg = mndBuildDropVnodeMsg(pMnode, pDnode, pDb, pVgroup); - if (pMsg == NULL) { - return -1; - } + if (pMsg == NULL) return -1; - if (mndTransAppendUndoAction(pTrans, &epset, TSDB_MSG_TYPE_DROP_VNODE_IN, sizeof(SDropVnodeMsg), pMsg) != 0) { + action.pCont = pMsg; + action.contLen = sizeof(SDropVnodeMsg); + action.msgType = TSDB_MSG_TYPE_DROP_VNODE_IN; + if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pMsg); return -1; } @@ -344,14 +353,14 @@ static int32_t mndSetUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SV static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) { SDbObj dbObj = {0}; - tstrncpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); - tstrncpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); + memcpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN); + memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN); dbObj.createdTime = taosGetTimestampMs(); dbObj.updateTime = dbObj.createdTime; dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN); - dbObj.hashMethod = 1; dbObj.cfgVersion = 1; dbObj.vgVersion = 1; + dbObj.hashMethod = 1; dbObj.cfg = (SDbCfg){.numOfVgroups = pCreate->numOfVgroups, .cacheBlockSize = pCreate->cacheBlockSize, .totalBlocks = pCreate->totalBlocks, @@ -359,8 +368,8 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat .daysToKeep0 = pCreate->daysToKeep0, .daysToKeep1 = pCreate->daysToKeep1, .daysToKeep2 = pCreate->daysToKeep2, - .minRows = pCreate->minRowsPerFileBlock, - .maxRows = pCreate->maxRowsPerFileBlock, + .minRows = pCreate->minRows, + .maxRows = pCreate->maxRows, .fsyncPeriod = pCreate->fsyncPeriod, .commitTime = pCreate->commitTime, .precision = pCreate->precision, @@ -396,29 +405,30 @@ static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreat mError("db:%s, failed to create since %s", pCreate->db, terrstr()); goto CREATE_DB_OVER; } + mDebug("trans:%d, used to create db:%s", pTrans->id, pCreate->db); - if (mndSetRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbRedoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbUndoLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbCommitLogs(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbRedoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } - if (mndSetUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { + if (mndSetCreateDbUndoActions(pMnode, pTrans, &dbObj, pVgroups) != 0) { mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); goto CREATE_DB_OVER; } @@ -447,8 +457,8 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0); pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); - pCreate->minRowsPerFileBlock = htonl(pCreate->minRowsPerFileBlock); - pCreate->maxRowsPerFileBlock = htonl(pCreate->maxRowsPerFileBlock); + pCreate->minRows = htonl(pCreate->minRows); + pCreate->maxRows = htonl(pCreate->maxRows); pCreate->commitTime = htonl(pCreate->commitTime); pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); @@ -456,7 +466,7 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db); if (pDb != NULL) { - sdbRelease(pMnode->pSdb, pDb); + mndReleaseDb(pMnode, pDb); if (pCreate->ignoreExist) { mDebug("db:%s, already exist, ignore exist is set", pCreate->db); return 0; @@ -485,57 +495,77 @@ static int32_t mndProcessCreateDbMsg(SMnodeMsg *pMsg) { } static int32_t mndSetDbCfgFromAlterDbMsg(SDbObj *pDb, SAlterDbMsg *pAlter) { - bool changed = false; + terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; if (pAlter->totalBlocks >= 0 && pAlter->totalBlocks != pDb->cfg.totalBlocks) { pDb->cfg.totalBlocks = pAlter->totalBlocks; - changed = true; + terrno = 0; } if (pAlter->daysToKeep0 >= 0 && pAlter->daysToKeep0 != pDb->cfg.daysToKeep0) { pDb->cfg.daysToKeep0 = pAlter->daysToKeep0; - changed = true; + terrno = 0; } if (pAlter->daysToKeep1 >= 0 && pAlter->daysToKeep1 != pDb->cfg.daysToKeep1) { pDb->cfg.daysToKeep1 = pAlter->daysToKeep1; - changed = true; + terrno = 0; } if (pAlter->daysToKeep2 >= 0 && pAlter->daysToKeep2 != pDb->cfg.daysToKeep2) { pDb->cfg.daysToKeep2 = pAlter->daysToKeep2; - changed = true; + terrno = 0; } if (pAlter->fsyncPeriod >= 0 && pAlter->fsyncPeriod != pDb->cfg.fsyncPeriod) { pDb->cfg.fsyncPeriod = pAlter->fsyncPeriod; - changed = true; + terrno = 0; } if (pAlter->walLevel >= 0 && pAlter->walLevel != pDb->cfg.walLevel) { pDb->cfg.walLevel = pAlter->walLevel; - changed = true; + terrno = 0; } if (pAlter->quorum >= 0 && pAlter->quorum != pDb->cfg.quorum) { pDb->cfg.quorum = pAlter->quorum; - changed = true; + terrno = 0; } if (pAlter->cacheLastRow >= 0 && pAlter->cacheLastRow != pDb->cfg.cacheLastRow) { pDb->cfg.cacheLastRow = pAlter->cacheLastRow; - changed = true; + terrno = 0; } - if (!changed) { - terrno = TSDB_CODE_MND_DB_OPTION_UNCHANGED; - return -1; - } + return terrno; +} + +static int32_t mndSetUpdateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { + SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; return 0; } +static int32_t mndSetUpdateDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { + SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetUpdateDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } + +static int32_t mndSetUpdateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } + +static int32_t mndSetUpdateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb, SDbObj *pNewDb) { return 0; } + static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbObj *pNewDb) { + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("db:%s, failed to update since %s", pOldDb->name, terrstr()); @@ -544,30 +574,41 @@ static int32_t mndUpdateDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pOldDb, SDbO mDebug("trans:%d, used to update db:%s", pTrans->id, pOldDb->name); - SSdbRaw *pRedoRaw = mndDbActionEncode(pNewDb); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetUpdateDbRedoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); - SSdbRaw *pUndoRaw = mndDbActionEncode(pOldDb); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetUpdateDbUndoLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; + } + + if (mndSetUpdateDbCommitLogs(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; + } + + if (mndSetUpdateDbRedoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; + } + + if (mndSetUpdateDbUndoActions(pMnode, pTrans, pOldDb, pNewDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto UPDATE_DB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + goto UPDATE_DB_OVER; } + code = 0; + +UPDATE_DB_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { @@ -610,46 +651,82 @@ static int32_t mndProcessAlterDbMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } +static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropDbUndoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdbRaw *pUndoRaw = mndDbActionEncode(pDb); + if (pUndoRaw == NULL) return -1; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdbRaw *pCommitRaw = mndDbActionEncode(pDb); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndSetDropDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; } + +static int32_t mndSetDropDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { return 0; } + static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pMsg, SDbObj *pDb) { + int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) { mError("db:%s, failed to drop since %s", pDb->name, terrstr()); return -1; } + mDebug("trans:%d, used to drop db:%s", pTrans->id, pDb->name); - SSdbRaw *pRedoRaw = mndDbActionEncode(pDb); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { - mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; } - sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); - SSdbRaw *pUndoRaw = mndDbActionEncode(pDb); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) { - mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropDbUndoLogs(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set undo log since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; } - sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); - SSdbRaw *pCommitRaw = mndDbActionEncode(pDb); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { - mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; + } + + if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; + } + + if (mndSetDropDbUndoActions(pMnode, pTrans, pDb) != 0) { + mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr()); + goto DROP_DB_OVER; } - sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - return -1; + goto DROP_DB_OVER; } + code = 0; + +DROP_DB_OVER: mndTransDrop(pTrans); - return 0; + return code; } static int32_t mndProcessDropDbMsg(SMnodeMsg *pMsg) { @@ -751,29 +828,27 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) { static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SSyncDbMsg *pSync = pMsg->rpcMsg.pCont; - - SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); if (pDb == NULL) { mError("db:%s, failed to process sync db msg since %s", pMsg->db, terrstr()); return -1; - } else { - mndReleaseDb(pMnode, pDb); - return 0; } + + mndReleaseDb(pMnode, pDb); + return 0; } static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SCompactDbMsg *pCompact = pMsg->rpcMsg.pCont; - - SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); if (pDb == NULL) { mError("db:%s, failed to process compact db msg since %s", pMsg->db, terrstr()); return -1; - } else { - mndReleaseDb(pMnode, pDb); - return 0; } + + mndReleaseDb(pMnode, pDb); + return 0; } static int32_t mndGetDbMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 493f20bc9af3319521fd602e2ed8e6648305ca22..9751771773c53fa55ec965a1fed8be02013f7588 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -18,10 +18,10 @@ #include "mndMnode.h" #include "mndShow.h" #include "mndTrans.h" -#include "ttime.h" #include "tep.h" +#include "ttime.h" -#define TSDB_DNODE_VER 1 +#define TSDB_DNODE_VER_NUMBER 1 #define TSDB_DNODE_RESERVE_SIZE 64 #define TSDB_CONFIG_OPTION_LEN 16 #define TSDB_CONIIG_VALUE_LEN 48 @@ -101,14 +101,14 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { SSdbRaw *pRaw = mndDnodeActionEncode(&dnodeObj); if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); + if (sdbSetRawStatus(pRaw, SDB_STATUS_READY) != 0) return -1; mDebug("dnode:%d, will be created while deploy sdb", dnodeObj.id); return sdbWrite(pMnode->pSdb, pRaw); } static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DNODE, TSDB_DNODE_VER_NUMBER, sizeof(SDnodeObj) + TSDB_DNODE_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -127,7 +127,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_DNODE_VER) { + if (sver != TSDB_DNODE_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to decode dnode since %s", terrstr()); return NULL; @@ -150,21 +150,8 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw) { static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) { mTrace("dnode:%d, perform insert action", pDnode->id); - - pDnode->rebootTime = 0; - pDnode->lastAccessTime = 0; - pDnode->accessTimes = 0; - pDnode->numOfMnodes = 0; - pDnode->numOfVnodes = 0; - pDnode->numOfQnodes = 0; - pDnode->numOfSupportMnodes = 0; - pDnode->numOfSupportVnodes = 0; - pDnode->numOfSupportQnodes = 0; - pDnode->numOfCores = 0; - pDnode->status = DND_STATUS_OFFLINE; pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED; snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port); - return 0; } @@ -225,7 +212,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeInReadyStatus(SMnode *pMnode, SDnodeObj *pDnode) { int64_t ms = taosGetTimestampMs(); int64_t interval = ABS(pDnode->lastAccessTime - ms); - if (interval > 3000 * pMnode->cfg.statusInterval) { + if (interval > 3500 * pMnode->cfg.statusInterval) { return false; } return true; @@ -267,12 +254,9 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, const SClusterCfg *pCfg) { return DND_REASON_STATUS_INTERVAL_NOT_MATCH; } - int64_t checkTime = 0; - char timestr[32] = "1970-01-01 00:00:00.00"; - (void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); - if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (checkTime != pCfg->checkTime)) { + if ((0 != strcasecmp(pCfg->timezone, pMnode->cfg.timezone)) && (pMnode->checkTime != pCfg->checkTime)) { mError("timezone [%s - %s] [%" PRId64 " - %" PRId64 "] cfg inconsistent", pCfg->timezone, pMnode->cfg.timezone, - pCfg->checkTime, checkTime); + pCfg->checkTime, pMnode->checkTime); return DND_REASON_TIME_ZONE_NOT_MATCH; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 16733cbf94bd93fc6193d6ca7bf0cd6c39e88f94..05c02ceac63b388eac1d808d5a7c07a7bc884df8 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -19,7 +19,8 @@ #include "mndShow.h" #include "mndTrans.h" -#define SDB_MNODE_VER 1 +#define TSDB_MNODE_VER_NUMBER 1 +#define TSDB_MNODE_RESERVE_SIZE 64 static int32_t mndCreateDefaultMnode(SMnode *pMnode); static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj); @@ -97,13 +98,14 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { } static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, SDB_MNODE_VER, sizeof(SMnodeObj)); + SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, TSDB_MNODE_VER_NUMBER, sizeof(SMnodeObj)); if (pRaw == NULL) return NULL; int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, pMnodeObj->id); SDB_SET_INT64(pRaw, dataPos, pMnodeObj->createdTime) SDB_SET_INT64(pRaw, dataPos, pMnodeObj->updateTime) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_MNODE_RESERVE_SIZE) return pRaw; } @@ -112,7 +114,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != SDB_MNODE_VER) { + if (sver != TSDB_MNODE_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to decode mnode since %s", terrstr()); return NULL; @@ -126,6 +128,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, pRow, dataPos, &pMnodeObj->id) SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->updateTime) + SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_MNODE_RESERVE_SIZE) return pRow; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 5caec6c78dc59bd061e96770c7817510c43a0f6f..c31a8c5b7d767e05c5a1850a6a808f6e0eda11c4 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -17,17 +17,10 @@ #include "mndTrans.h" #include "mndSync.h" -#define TSDB_TRANS_VER 1 +#define TSDB_TRANS_VER_NUMBER 1 #define TSDB_TRN_ARRAY_SIZE 8 #define TSDB_TRN_RESERVE_SIZE 64 -typedef struct { - SEpSet epSet; - int8_t msgType; - int32_t contLen; - void *pCont; -} STransAction; - static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); @@ -37,11 +30,11 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); static void mndTransSetRpcHandle(STrans *pTrans, void *rpcHandle); static void mndTransSendRpcRsp(STrans *pTrans, int32_t code); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); -static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont); +static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); static void mndTransDropLogs(SArray *pArray); static void mndTransDropActions(SArray *pArray); static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray); -static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray); +static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray); static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans); @@ -100,7 +93,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { rawDataLen += (sizeof(STransAction) + pAction->contLen); } - SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER, rawDataLen); + SSdbRaw *pRaw = sdbAllocRaw(SDB_TRANS, TSDB_TRANS_VER_NUMBER, rawDataLen); if (pRaw == NULL) { mError("trans:%d, failed to alloc raw since %s", pTrans->id, terrstr()); return NULL; @@ -167,7 +160,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { return NULL; } - if (sver != TSDB_TRANS_VER) { + if (sver != TSDB_TRANS_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; mError("failed to get check soft ver from raw:%p since %s", pRaw, terrstr()); return NULL; @@ -343,10 +336,8 @@ char *mndTransStageStr(ETrnStage stage) { return "commit"; case TRN_STAGE_ROLLBACK: return "rollback"; - case TRN_STAGE_RETRY: - return "retry"; case TRN_STAGE_OVER: - return "stop"; + return "over"; default: return "undefined"; } @@ -388,7 +379,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, void *rpcHandle) { return NULL; } - mDebug("trans:%d, data:%p is created", pTrans->id, pTrans); + mDebug("trans:%d, is created", pTrans->id); return pTrans; } @@ -417,7 +408,7 @@ void mndTransDrop(STrans *pTrans) { mndTransDropActions(pTrans->redoActions); mndTransDropActions(pTrans->undoActions); - mDebug("trans:%d, data:%p is dropped", pTrans->id, pTrans); + // mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); tfree(pTrans); } @@ -459,10 +450,8 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw) { return code; } -static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { - STransAction action = {.epSet = *pEpSet, .msgType = msgType, .contLen = contLen, .pCont = pCont}; - - void *ptr = taosArrayPush(pArray, &action); +static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction) { + void *ptr = taosArrayPush(pArray, pAction); if (ptr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -471,16 +460,12 @@ static int32_t mndTransAppendAction(SArray *pArray, SEpSet *pEpSet, int8_t msgTy return 0; } -int32_t mndTransAppendRedoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { - int32_t code = mndTransAppendAction(pTrans->redoActions, pEpSet, msgType, contLen, pCont); - mTrace("trans:%d, msg:%s len:%d append to redo actions", pTrans->id, taosMsg[msgType], contLen); - return code; +int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction) { + return mndTransAppendAction(pTrans->redoActions, pAction); } -int32_t mndTransAppendUndoAction(STrans *pTrans, SEpSet *pEpSet, int8_t msgType, int32_t contLen, void *pCont) { - int32_t code = mndTransAppendAction(pTrans->undoActions, pEpSet, msgType, contLen, pCont); - mTrace("trans:%d, msg:%s len:%d append to undo actions", pTrans->id, taosMsg[msgType], contLen); - return code; +int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction) { + return mndTransAppendAction(pTrans->undoActions, pAction); } int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { @@ -493,7 +478,7 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { } sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mTrace("trans:%d, start sync", pTrans->id); + mTrace("trans:%d, sync to other nodes", pTrans->id); int32_t code = mndSyncPropose(pMnode, pRaw); if (code != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); @@ -533,7 +518,7 @@ int32_t mndTransCommit(SMnode *pMnode, STrans *pTrans) { sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); if (taosArrayGetSize(pTrans->commitLogs) != 0) { - mTrace("trans:%d, start sync", pTrans->id); + mTrace("trans:%d, sync to other nodes", pTrans->id); int32_t code = mndSyncPropose(pMnode, pRaw); if (code != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); @@ -563,7 +548,7 @@ int32_t mndTransRollback(SMnode *pMnode, STrans *pTrans) { } sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); - mTrace("trans:%d, start sync", pTrans->id); + mTrace("trans:%d, sync to other nodes", pTrans->id); int32_t code = mndSyncPropose(pMnode, pRaw); if (code != 0) { mError("trans:%d, failed to sync since %s", pTrans->id, terrstr()); @@ -596,6 +581,50 @@ void mndTransApply(SMnode *pMnode, SSdbRaw *pRaw, STransMsg *pMsg, int32_t code) // todo } +void mndTransHandleActionRsp(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + int64_t sig = (int64_t)(pMsg->rpcMsg.ahandle); + int32_t transId = (int32_t)(sig >> 32); + int32_t action = (int32_t)((sig << 32) >> 32); + + STrans *pTrans = mndAcquireTrans(pMnode, transId); + if (pTrans == NULL) { + mError("trans:%d, failed to get transId from vnode rsp since %s", transId, terrstr()); + goto HANDLE_ACTION_RSP_OVER; + } + + SArray *pArray = NULL; + if (pTrans->stage == TRN_STAGE_EXECUTE) { + pArray = pTrans->redoActions; + } else if (pTrans->stage == TRN_STAGE_ROLLBACK) { + pArray = pTrans->undoActions; + } else { + } + + if (pArray == NULL) { + mError("trans:%d, invalid trans stage:%s", transId, mndTransStageStr(pTrans->stage)); + goto HANDLE_ACTION_RSP_OVER; + } + + int32_t actionNum = taosArrayGetSize(pTrans->redoActions); + if (action < 0 || action > actionNum) { + mError("trans:%d, invalid action:%d", transId, action); + goto HANDLE_ACTION_RSP_OVER; + } + + STransAction *pAction = taosArrayGet(pArray, action); + if (pAction != NULL) { + pAction->msgReceived = 1; + pAction->errCode = pMsg->code; + } + + mDebug("trans:%d, action:%d response is received, code:0x%x", transId, action, pMsg->code); + mndTransExecute(pMnode, pTrans); + +HANDLE_ACTION_RSP_OVER: + mndReleaseTrans(pMnode, pTrans); +} + static int32_t mndTransExecuteLogs(SMnode *pMnode, SArray *pArray) { SSdb *pSdb = pMnode->pSdb; int32_t arraySize = taosArrayGetSize(pArray); @@ -618,7 +647,7 @@ static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans) { if (code != 0) { mError("trans:%d, failed to execute redo logs since %s", pTrans->id, terrstr()) } else { - mTrace("trans:%d, execute redo logs finished", pTrans->id) + mDebug("trans:%d, execute redo logs finished", pTrans->id) } } @@ -632,7 +661,7 @@ static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans) { if (code != 0) { mError("trans:%d, failed to execute undo logs since %s", pTrans->id, terrstr()) } else { - mTrace("trans:%d, execute undo logs finished", pTrans->id) + mDebug("trans:%d, execute undo logs finished", pTrans->id) } } @@ -646,47 +675,70 @@ static int32_t mndTransExecuteCommitLogs(SMnode *pMnode, STrans *pTrans) { if (code != 0) { mError("trans:%d, failed to execute commit logs since %s", pTrans->id, terrstr()) } else { - mTrace("trans:%d, execute commit logs finished", pTrans->id) + mDebug("trans:%d, execute commit logs finished", pTrans->id) } } return code; } -static int32_t mndTransExecuteActions(SMnode *pMnode, SArray *pArray) { -#if 0 - int32_t arraySize = taosArrayGetSize(pArray); - for (int32_t i = 0; i < arraySize; ++i) { - STransAction *pAction = taosArrayGet(pArray, i); +static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { + int32_t numOfActions = taosArrayGetSize(pArray); + if (numOfActions == 0) return 0; + + for (int32_t action = 0; action < numOfActions; ++action) { + STransAction *pAction = taosArrayGet(pArray, action); + if (pAction == NULL) continue; + if (pAction->msgSent) continue; + + int64_t signature = pTrans->id; + signature = (signature << 32); + signature += action; - SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen}; + SRpcMsg rpcMsg = {.msgType = pAction->msgType, .contLen = pAction->contLen, .ahandle = (void *)signature}; rpcMsg.pCont = rpcMallocCont(pAction->contLen); if (rpcMsg.pCont == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); + + pAction->msgSent = 1; + pAction->msgReceived = 0; + pAction->errCode = 0; + + mDebug("trans:%d, action:%d is sent", pTrans->id, action); mndSendMsgToDnode(pMnode, &pAction->epSet, &rpcMsg); } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; -#else - return 0; -#endif + int32_t numOfReceivedMsgs = 0; + int32_t errorCode = 0; + for (int32_t action = 0; action < numOfActions; ++action) { + STransAction *pAction = taosArrayGet(pArray, action); + if (pAction == NULL) continue; + if (pAction->msgSent && pAction->msgReceived) { + numOfReceivedMsgs++; + if (pAction->errCode != 0) { + errorCode = pAction->errCode; + } + } + } + + if (numOfReceivedMsgs == numOfActions) { + mDebug("trans:%d, all %d actions executed, code:0x%x", pTrans->id, numOfActions, errorCode); + terrno = errorCode; + return errorCode; + } else { + return TSDB_CODE_MND_ACTION_IN_PROGRESS; + } } static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { - if (taosArrayGetSize(pTrans->redoActions) <= 0) return 0; - - mTrace("trans:%d, start to execute redo actions", pTrans->id); - return mndTransExecuteActions(pMnode, pTrans->redoActions); + return mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); } static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { - if (taosArrayGetSize(pTrans->undoActions) <= 0) return 0; - - mTrace("trans:%d, start to execute undo actions", pTrans->id); - return mndTransExecuteActions(pMnode, pTrans->undoActions); + return mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); } static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { @@ -694,7 +746,7 @@ static int32_t mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { if (code == 0) { pTrans->stage = TRN_STAGE_EXECUTE; - mTrace("trans:%d, stage from prepare to execute", pTrans->id); + mDebug("trans:%d, stage from prepare to execute", pTrans->id); } else { pTrans->stage = TRN_STAGE_ROLLBACK; mError("trans:%d, stage from prepare to rollback since %s", pTrans->id, terrstr()); @@ -708,17 +760,17 @@ static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) { if (code == 0) { pTrans->stage = TRN_STAGE_COMMIT; - mTrace("trans:%d, stage from execute to commit", pTrans->id); + mDebug("trans:%d, stage from execute to commit", pTrans->id); } else if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) { - mTrace("trans:%d, stage keep on execute since %s", pTrans->id, terrstr(code)); + mDebug("trans:%d, stage keep on execute since %s", pTrans->id, tstrerror(code)); return code; } else { if (pTrans->policy == TRN_POLICY_ROLLBACK) { pTrans->stage = TRN_STAGE_ROLLBACK; mError("trans:%d, stage from execute to rollback since %s", pTrans->id, terrstr()); } else { - pTrans->stage = TRN_STAGE_RETRY; - mError("trans:%d, stage from execute to retry since %s", pTrans->id, terrstr()); + pTrans->stage = TRN_STAGE_EXECUTE; + mError("trans:%d, stage keep on execute since %s", pTrans->id, terrstr()); } } @@ -726,29 +778,16 @@ static int32_t mndTransPerformExecuteStage(SMnode *pMnode, STrans *pTrans) { } static int32_t mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteCommitLogs(pMnode, pTrans); - - if (code == 0) { - pTrans->stage = TRN_STAGE_OVER; - mTrace("trans:%d, commit stage finished", pTrans->id); - } else { - if (pTrans->policy == TRN_POLICY_ROLLBACK) { - pTrans->stage = TRN_STAGE_ROLLBACK; - mError("trans:%d, stage from commit to rollback since %s", pTrans->id, terrstr()); - } else { - pTrans->stage = TRN_STAGE_RETRY; - mError("trans:%d, stage from commit to retry since %s", pTrans->id, terrstr()); - } - } - - return code; + mndTransExecuteCommitLogs(pMnode, pTrans); + pTrans->stage = TRN_STAGE_OVER; + return 0; } static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteUndoActions(pMnode, pTrans); if (code == 0) { - mTrace("trans:%d, rollbacked", pTrans->id); + mDebug("trans:%d, rollbacked", pTrans->id); } else { pTrans->stage = TRN_STAGE_ROLLBACK; mError("trans:%d, stage keep on rollback since %s", pTrans->id, terrstr()); @@ -757,20 +796,6 @@ static int32_t mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { return code; } -static int32_t mndTransPerformRetryStage(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteRedoActions(pMnode, pTrans); - - if (code == 0) { - pTrans->stage = TRN_STAGE_COMMIT; - mTrace("trans:%d, stage from retry to commit", pTrans->id); - } else { - pTrans->stage = TRN_STAGE_RETRY; - mError("trans:%d, stage keep on retry since %s", pTrans->id, terrstr()); - } - - return code; -} - static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { int32_t code = 0; @@ -785,7 +810,7 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { case TRN_STAGE_COMMIT: code = mndTransCommit(pMnode, pTrans); if (code == 0) { - code = mndTransPerformCommitStage(pMnode, pTrans); + mndTransPerformCommitStage(pMnode, pTrans); } break; case TRN_STAGE_ROLLBACK: @@ -794,9 +819,6 @@ static void mndTransExecute(SMnode *pMnode, STrans *pTrans) { code = mndTransRollback(pMnode, pTrans); } break; - case TRN_STAGE_RETRY: - code = mndTransPerformRetryStage(pMnode, pTrans); - break; default: mndTransSendRpcRsp(pTrans, 0); return; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 58d687a14865ec7fe104457d287115e4bc0737f0..eabf2df958bb6ba500cfa4766cf11371568b3a9d 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -20,7 +20,7 @@ #include "mndTrans.h" #include "tkey.h" -#define TSDB_USER_VER 1 +#define TSDB_USER_VER_NUMBER 1 #define TSDB_USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); @@ -94,7 +94,7 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { } static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, TSDB_USER_VER_NUMBER, sizeof(SUserObj) + TSDB_USER_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -114,7 +114,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_USER_VER) { + if (sver != TSDB_USER_VER_NUMBER) { mError("failed to decode user since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index e99fea200b944639aca3fb5bcb33623033c7915d..eeca8c9546e22f8f83962e833487e238385b06eb 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -21,7 +21,7 @@ #include "mndShow.h" #include "mndTrans.h" -#define TSDB_VGROUP_VER_NUM 1 +#define TSDB_VGROUP_VER_NUMBER 1 #define TSDB_VGROUP_RESERVE_SIZE 64 static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); @@ -70,7 +70,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { void mndCleanupVgroup(SMnode *pMnode) {} SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { - SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUM, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUMBER, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); if (pRaw == NULL) return NULL; int32_t dataPos = 0; @@ -98,7 +98,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != TSDB_VGROUP_VER_NUM) { + if (sver != TSDB_VGROUP_VER_NUMBER) { mError("failed to decode vgroup since %s", terrstr()); terrno = TSDB_CODE_SDB_INVALID_DATA_VER; return NULL; @@ -142,14 +142,20 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOldVgroup, SVgObj *pNe mTrace("vgId:%d, perform update action", pOldVgroup->vgId); pOldVgroup->updateTime = pNewVgroup->updateTime; pOldVgroup->version = pNewVgroup->version; + pOldVgroup->hashBegin = pNewVgroup->hashBegin; + pOldVgroup->hashEnd = pNewVgroup->hashEnd; pOldVgroup->replica = pNewVgroup->replica; memcpy(pOldVgroup->vnodeGid, pNewVgroup->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); return 0; } SVgObj *mndAcquireVgroup(SMnode *pMnode, int32_t vgId) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_VGROUP, &vgId); + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = sdbAcquire(pSdb, SDB_VGROUP, &vgId); + if (pVgroup == NULL) { + terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST; + } + return pVgroup; } void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { @@ -158,16 +164,17 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { } SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SCreateVnodeMsg *pCreate = malloc(sizeof(SCreateVnodeMsg)); + SCreateVnodeMsg *pCreate = calloc(1, sizeof(SCreateVnodeMsg)); if (pCreate == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pCreate->dnodeId = htonl(pDnode->id); pCreate->vgId = htonl(pVgroup->vgId); + pCreate->dnodeId = htonl(pDnode->id); memcpy(pCreate->db, pDb->name, TSDB_FULL_DB_NAME_LEN); pCreate->dbUid = htobe64(pDb->uid); + pCreate->vgVersion = htonl(pVgroup->version); pCreate->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); pCreate->totalBlocks = htonl(pDb->cfg.totalBlocks); pCreate->daysPerFile = htonl(pDb->cfg.daysPerFile); @@ -193,7 +200,6 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pVgidDnode == NULL) { free(pCreate); - terrno = TSDB_CODE_MND_APP_ERROR; return NULL; } @@ -217,7 +223,7 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb } SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SDropVnodeMsg *pDrop = malloc(sizeof(SDropVnodeMsg)); + SDropVnodeMsg *pDrop = calloc(1, sizeof(SDropVnodeMsg)); if (pDrop == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; @@ -269,7 +275,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { return -1; } - int32_t alloceVgroups = 0; + int32_t allocedVgroups = 0; int32_t maxVgId = sdbGetMaxId(pMnode->pSdb, SDB_VGROUP); uint32_t hashMin = 0; uint32_t hashMax = UINT32_MAX; @@ -281,7 +287,6 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { pVgroup->createdTime = taosGetTimestampMs(); pVgroup->updateTime = pVgroups->createdTime; pVgroup->version = 1; - pVgroup->dbUid = pDb->uid; pVgroup->hashBegin = hashMin + hashInterval * v; if (v == pDb->cfg.numOfVgroups - 1) { pVgroup->hashEnd = hashMax; @@ -290,6 +295,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { } memcpy(pVgroup->dbName, pDb->name, TSDB_FULL_DB_NAME_LEN); + pVgroup->dbUid = pDb->uid; pVgroup->replica = pDb->cfg.replications; if (mndGetAvailableDnode(pMnode, pVgroup) != 0) { @@ -298,22 +304,25 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { return -1; } - alloceVgroups++; + allocedVgroups++; } *ppVgroups = pVgroups; return 0; } -static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCreateVnodeRsp(SMnodeMsg *pMsg) { + mndTransHandleActionRsp(pMsg); + return 0; +} + static int32_t mndProcessAlterVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessDropVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessSyncVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessCompactVnodeRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pReplica, int32_t *pNumOfVgroups) { - SSdb *pSdb = pMnode->pSdb; - + SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -329,7 +338,7 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; - if (strcmp(pVgroup->dbName, dbName) == 0) { + if (pVgroup->dbUid == pDb->uid) { replica = MAX(replica, pVgroup->replica); numOfVgroups++; } @@ -441,11 +450,25 @@ static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter) { } static int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { - if (dnodeId == 0) { - return 0; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfVnodes = 0; + void *pIter = NULL; + + while (1) { + SVgObj *pVgroup = NULL; + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + if (pVgroup->vnodeGid[v].dnodeId == dnodeId) { + numOfVnodes++; + } + } + + sdbRelease(pSdb, pVgroup); } - return 0; + return numOfVnodes; } static int32_t mndGetVnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 24155a4e5329f73fc950e25ef23b942b55a11bf0..fb0b95dc4a696f9a7b2751f3735d73d059838c60 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -225,7 +225,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { } return 0; -} +} SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { mDebug("start to open mnode in %s", path); @@ -237,6 +237,9 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return NULL; } + char timestr[24] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &pMnode->checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + pMnode->pSteps = taosArrayInit(24, sizeof(SMnodeStep)); if (pMnode->pSteps == NULL) { free(pMnode); diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index e37559808e2b5242354be6187c07fc49a732a940..5a0020199f0652376de73ec5c243301b4ac2371f 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -27,12 +27,12 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) { pRaw->sver = sver; pRaw->dataLen = dataLen; - mTrace("raw:%p, is created, len:%d", pRaw, dataLen); + // mTrace("raw:%p, is created, len:%d", pRaw, dataLen); return pRaw; } void sdbFreeRaw(SSdbRaw *pRaw) { - mTrace("raw:%p, is freed", pRaw); + // mTrace("raw:%p, is freed", pRaw); free(pRaw); } diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index d8d736f3d89854dabe5ae2a0f814d08ca0265ec0..f564afd613fc68519318ba6115f254d18aa25b17 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -16,17 +16,13 @@ #include "vnodeDef.h" int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { - SVnodeReq *pVnodeReq; - switch (pMsg->msgType) { - case TSDB_MSG_TYPE_MQ_SET: + case TSDB_MSG_TYPE_MQ_SET_CUR: if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) { // TODO: handle error } break; } - - void *pBuf = pMsg->pCont; return 0; } diff --git a/source/dnode/vnode/tq/CMakeLists.txt b/source/dnode/vnode/tq/CMakeLists.txt index 536e97d5f79b58442b2d0a6af5758bb4ec854fc1..8d59c7b07a9c3935187ba6b644d7f95f88179776 100644 --- a/source/dnode/vnode/tq/CMakeLists.txt +++ b/source/dnode/vnode/tq/CMakeLists.txt @@ -11,6 +11,7 @@ target_link_libraries( PUBLIC wal PUBLIC os PUBLIC util + PUBLIC common ) if(${BUILD_TEST}) diff --git a/source/dnode/vnode/tq/inc/tqInt.h b/source/dnode/vnode/tq/inc/tqInt.h index 022b5998162b907382643d1e01a8cf0b05596696..5685a29d03e7dc15a974d9728d0c6ec07d4cceb3 100644 --- a/source/dnode/vnode/tq/inc/tqInt.h +++ b/source/dnode/vnode/tq/inc/tqInt.h @@ -17,11 +17,20 @@ #define _TD_TQ_INT_H_ #include "tq.h" - +#include "tlog.h" #ifdef __cplusplus extern "C" { #endif +extern int32_t tqDebugFlag; + +#define tqFatal(...) { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); }} +#define tqError(...) { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); }} +#define tqWarn(...) { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", 255, __VA_ARGS__); }} +#define tqInfo(...) { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", 255, __VA_ARGS__); }} +#define tqDebug(...) { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} +#define tqTrace(...) { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} + // create persistent storage for meta info such as consuming offset // return value > 0: cgId // return value <= 0: error code diff --git a/source/dnode/vnode/tq/inc/tqMetaStore.h b/source/dnode/vnode/tq/inc/tqMetaStore.h index 5bcedaed748d7d0668181183d09de7470ff630e2..ef71d8bf145ae535edf7740e864661e576078814 100644 --- a/source/dnode/vnode/tq/inc/tqMetaStore.h +++ b/source/dnode/vnode/tq/inc/tqMetaStore.h @@ -17,7 +17,7 @@ #define _TQ_META_STORE_H_ #include "os.h" -#include "tq.h" +#include "tqInt.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/tq/src/tq.c index 1a27870a1b738d52f1c142f010e8e7e44abce50d..d8dfe4ddcf16e0f91988ae6a637376f709f73104 100644 --- a/source/dnode/vnode/tq/src/tq.c +++ b/source/dnode/vnode/tq/src/tq.c @@ -24,89 +24,80 @@ // handle management message // -int tqGetgHandleSSize(const STqGroupHandle* gHandle); -int tqBufHandleSSize(); -int tqBufItemSSize(); +int tqGroupSSize(const STqGroup* pGroup); +int tqTopicSSize(); +int tqItemSSize(); -STqGroupHandle* tqFindHandle(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - STqGroupHandle* gHandle; - return NULL; -} +void* tqSerializeListHandle(STqList* listHandle, void* ptr); +void* tqSerializeTopic(STqTopic* pTopic, void* ptr); +void* tqSerializeItem(STqMsgItem* pItem, void* ptr); -void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr); -void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr); -void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr); - -const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle); -const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem); +const void* tqDeserializeTopic(const void* pBytes, STqTopic* pTopic); +const void* tqDeserializeItem(const void* pBytes, STqMsgItem* pItem); STQ* tqOpen(const char* path, STqCfg* tqConfig, STqLogReader* tqLogReader, SMemAllocatorFactory* allocFac) { STQ* pTq = malloc(sizeof(STQ)); if (pTq == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } pTq->path = strdup(path); pTq->tqConfig = tqConfig; pTq->tqLogReader = tqLogReader; pTq->tqMemRef.pAlloctorFactory = allocFac; - // pTq->tqMemRef.pAllocator = allocFac->create(allocFac); + pTq->tqMemRef.pAllocator = allocFac->create(allocFac); if (pTq->tqMemRef.pAllocator == NULL) { - // TODO + // TODO: error code of buffer pool } - pTq->tqMeta = - tqStoreOpen(path, (FTqSerialize)tqSerializeGroupHandle, (FTqDeserialize)tqDeserializeGroupHandle, free, 0); + pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeGroup, (FTqDeserialize)tqDeserializeGroup, free, 0); if (pTq->tqMeta == NULL) { // TODO: free STQ return NULL; } return pTq; } - -void tqClose(STQ*pTq) { +void tqClose(STQ* pTq) { // TODO } -static int tqProtoCheck(TmqMsgHead *pMsg) { - return pMsg->protoVer == 0; -} +static int tqProtoCheck(STqMsgHead* pMsg) { return pMsg->protoVer == 0; } -static int tqAckOneTopic(STqBufferHandle* bHandle, TmqOneAck* pAck, STqQueryMsg** ppQuery) { +static int tqAckOneTopic(STqTopic* pTopic, STqOneAck* pAck, STqQueryMsg** ppQuery) { // clean old item and move forward int32_t consumeOffset = pAck->consumeOffset; int idx = consumeOffset % TQ_BUFFER_SIZE; - ASSERT(bHandle->buffer[idx].content && bHandle->buffer[idx].executor); - tfree(bHandle->buffer[idx].content); + ASSERT(pTopic->buffer[idx].content && pTopic->buffer[idx].executor); + tfree(pTopic->buffer[idx].content); if (1 /* TODO: need to launch new query */) { STqQueryMsg* pNewQuery = malloc(sizeof(STqQueryMsg)); if (pNewQuery == NULL) { - // TODO: memory insufficient + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } // TODO: lock executor - pNewQuery->exec->executor = bHandle->buffer[idx].executor; // TODO: read from wal and assign to src - pNewQuery->exec->src = 0; - pNewQuery->exec->dest = &bHandle->buffer[idx]; - pNewQuery->next = *ppQuery; - *ppQuery = pNewQuery; + /*pNewQuery->exec->executor = pTopic->buffer[idx].executor;*/ + /*pNewQuery->exec->src = 0;*/ + /*pNewQuery->exec->dest = &pTopic->buffer[idx];*/ + /*pNewQuery->next = *ppQuery;*/ + /**ppQuery = pNewQuery;*/ } return 0; } -static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) { +static int tqAck(STqGroup* pGroup, STqAcks* pAcks) { int32_t ackNum = pAcks->ackNum; - TmqOneAck* acks = pAcks->acks; + STqOneAck* acks = pAcks->acks; // double ptr for acks and list - int i = 0; - STqListHandle* node = gHandle->head; - int ackCnt = 0; - STqQueryMsg* pQuery = NULL; + int i = 0; + STqList* node = pGroup->head; + int ackCnt = 0; + STqQueryMsg* pQuery = NULL; while (i < ackNum && node->next) { - if (acks[i].topicId == node->next->bufHandle.topicId) { + if (acks[i].topicId == node->next->topic.topicId) { ackCnt++; - tqAckOneTopic(&node->next->bufHandle, &acks[i], &pQuery); - } else if (acks[i].topicId < node->next->bufHandle.topicId) { + tqAckOneTopic(&node->next->topic, &acks[i], &pQuery); + } else if (acks[i].topicId < node->next->topic.topicId) { i++; } else { node = node->next; @@ -118,52 +109,56 @@ static int tqAck(STqGroupHandle* gHandle, TmqAcks* pAcks) { return ackCnt; } -static int tqCommitTCGroup(STqGroupHandle* handle) { +static int tqCommitGroup(STqGroup* pGroup) { // persist modification into disk return 0; } -int tqCreateTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroupHandle** handle) { +int tqCreateGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId, STqGroup** ppGroup) { // create in disk - STqGroupHandle* gHandle = (STqGroupHandle*)malloc(sizeof(STqGroupHandle)); - if (gHandle == NULL) { + STqGroup* pGroup = (STqGroup*)malloc(sizeof(STqGroup)); + if (pGroup == NULL) { // TODO return -1; } - memset(gHandle, 0, sizeof(STqGroupHandle)); + *ppGroup = pGroup; + memset(pGroup, 0, sizeof(STqGroup)); return 0; } -STqGroupHandle* tqOpenTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { - STqGroupHandle* gHandle = tqHandleGet(pTq->tqMeta, cId); - if (gHandle == NULL) { - int code = tqCreateTCGroup(pTq, topicId, cgId, cId, &gHandle); - if (code != 0) { +STqGroup* tqOpenGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { + STqGroup* pGroup = tqHandleGet(pTq->tqMeta, cId); + if (pGroup == NULL) { + int code = tqCreateGroup(pTq, topicId, cgId, cId, &pGroup); + if (code < 0) { // TODO return NULL; } + tqHandleMovePut(pTq->tqMeta, cId, pGroup); } + ASSERT(pGroup); - // create - // open - return gHandle; + return pGroup; } -int tqCloseTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { return 0; } +int tqCloseGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { + // TODO + return 0; +} -int tqDropTCGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { +int tqDropGroup(STQ* pTq, int64_t topicId, int64_t cgId, int64_t cId) { // delete from disk return 0; } -static int tqFetch(STqGroupHandle* gHandle, void** msg) { - STqListHandle* head = gHandle->head; - STqListHandle* node = head; - int totSize = 0; +static int tqFetch(STqGroup* pGroup, void** msg) { + STqList* head = pGroup->head; + STqList* node = head; + int totSize = 0; // TODO: make it a macro int sizeLimit = 4 * 1024; - TmqMsgContent* buffer = malloc(sizeLimit); + STqMsgContent* buffer = malloc(sizeLimit); if (buffer == NULL) { // TODO:memory insufficient return -1; @@ -172,25 +167,25 @@ static int tqFetch(STqGroupHandle* gHandle, void** msg) { // until all topic iterated or msgs over sizeLimit while (node->next) { node = node->next; - STqBufferHandle* bufHandle = &node->bufHandle; - int idx = bufHandle->nextConsumeOffset % TQ_BUFFER_SIZE; - if (bufHandle->buffer[idx].content != NULL && bufHandle->buffer[idx].offset == bufHandle->nextConsumeOffset) { - totSize += bufHandle->buffer[idx].size; + STqTopic* topicHandle = &node->topic; + int idx = topicHandle->nextConsumeOffset % TQ_BUFFER_SIZE; + if (topicHandle->buffer[idx].content != NULL && topicHandle->buffer[idx].offset == topicHandle->nextConsumeOffset) { + totSize += topicHandle->buffer[idx].size; if (totSize > sizeLimit) { void* ptr = realloc(buffer, totSize); if (ptr == NULL) { - totSize -= bufHandle->buffer[idx].size; + totSize -= topicHandle->buffer[idx].size; // TODO:memory insufficient // return msgs already copied break; } } - *((int64_t*)buffer) = bufHandle->topicId; + *((int64_t*)buffer) = topicHandle->topicId; buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - *((int64_t*)buffer) = bufHandle->buffer[idx].size; + *((int64_t*)buffer) = topicHandle->buffer[idx].size; buffer = POINTER_SHIFT(buffer, sizeof(int64_t)); - memcpy(buffer, bufHandle->buffer[idx].content, bufHandle->buffer[idx].size); - buffer = POINTER_SHIFT(buffer, bufHandle->buffer[idx].size); + memcpy(buffer, topicHandle->buffer[idx].content, topicHandle->buffer[idx].size); + buffer = POINTER_SHIFT(buffer, topicHandle->buffer[idx].size); if (totSize > sizeLimit) { break; } @@ -199,11 +194,19 @@ static int tqFetch(STqGroupHandle* gHandle, void** msg) { return totSize; } -STqGroupHandle* tqGetGroupHandle(STQ* pTq, int64_t cId) { return NULL; } - -int tqLaunchQuery(STqGroupHandle* gHandle) { return 0; } +STqGroup* tqGetGroup(STQ* pTq, int64_t clientId) { return tqHandleGet(pTq->tqMeta, clientId); } -int tqSendLaunchQuery(STqGroupHandle* gHandle) { return 0; } +int tqSendLaunchQuery(STqMsgItem* bufItem, int64_t offset) { + if (tqQueryExecuting(bufItem->status)) { + return 0; + } + bufItem->status = 1; + // load data from wal or buffer pool + // put into exec + // send exec into non blocking queue + // when query finished, put into buffer pool + return 0; +} /*int tqMoveOffsetToNext(TqGroupHandle* gHandle) {*/ /*return 0;*/ @@ -220,23 +223,96 @@ int tqCommit(STQ* pTq) { return 0; } -int tqSetCursor(STQ* pTq, void* msg) { +int tqBufferSetOffset(STqTopic* pTopic, int64_t offset) { + int code; + memset(pTopic->buffer, 0, sizeof(pTopic->buffer)); + // launch query + for (int i = offset; i < offset + TQ_BUFFER_SIZE; i++) { + int pos = i % TQ_BUFFER_SIZE; + code = tqSendLaunchQuery(&pTopic->buffer[pos], offset); + if (code < 0) { + // TODO: error handling + } + } + // set offset + pTopic->nextConsumeOffset = offset; + pTopic->floatingCursor = offset; + return 0; +} + +STqTopic* tqFindTopic(STqGroup* pGroup, int64_t topicId) { + // TODO + return NULL; +} + +int tqSetCursor(STQ* pTq, STqSetCurReq* pMsg) { + int code; + int64_t clientId = pMsg->head.clientId; + int64_t topicId = pMsg->topicId; + int64_t offset = pMsg->offset; + STqGroup* gHandle = tqGetGroup(pTq, clientId); + if (gHandle == NULL) { + // client not connect + return -1; + } + STqTopic* topicHandle = tqFindTopic(gHandle, topicId); + if (topicHandle == NULL) { + return -1; + } + if (pMsg->offset == topicHandle->nextConsumeOffset) { + return 0; + } + // TODO: check log last version + + code = tqBufferSetOffset(topicHandle, offset); + if (code < 0) { + // set error code + return -1; + } + + return 0; +} + +int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { + int64_t clientId = pMsg->head.clientId; + STqGroup* pGroup = tqGetGroup(pTq, clientId); + if (pGroup == NULL) { + terrno = TSDB_CODE_TQ_GROUP_NOT_SET; + return -1; + } + + STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; + int numOfMsgs = tqFetch(pGroup, (void**)&pRsp->msgs); + if (numOfMsgs < 0) { + return -1; + } + if (numOfMsgs == 0) { + // most recent data has been fetched + + // enable timer for blocking wait + // once new data written during wait time + // launch query and response + } + + // fetched a num of msgs, rpc response + return 0; } +#if 0 int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { - if (!tqProtoCheck((TmqMsgHead*)pMsg)) { + if (!tqProtoCheck((STqMsgHead*)pMsg)) { // proto version invalid return -1; } - int64_t clientId = pMsg->head.clientId; - STqGroupHandle* gHandle = tqGetGroupHandle(pTq, clientId); - if (gHandle == NULL) { + int64_t clientId = pMsg->head.clientId; + STqGroup* pGroup = tqGetGroup(pTq, clientId); + if (pGroup == NULL) { // client not connect return -1; } if (pMsg->acks.ackNum != 0) { - if (tqAck(gHandle, &pMsg->acks) != 0) { + if (tqAck(pGroup, &pMsg->acks) != 0) { // ack not success return -1; } @@ -244,22 +320,23 @@ int tqConsume(STQ* pTq, STqConsumeReq* pMsg) { STqConsumeRsp* pRsp = (STqConsumeRsp*)pMsg; - if (tqFetch(gHandle, (void**)&pRsp->msgs) <= 0) { + if (tqFetch(pGroup, (void**)&pRsp->msgs) <= 0) { // fetch error return -1; } // judge and launch new query - if (tqLaunchQuery(gHandle)) { - // launch query error - return -1; - } + /*if (tqSendLaunchQuery(gHandle)) {*/ + // launch query error + /*return -1;*/ + /*}*/ return 0; } +#endif -int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** ppHead) { +int tqSerializeGroup(const STqGroup* pGroup, STqSerializedHead** ppHead) { // calculate size - int sz = tqGetgHandleSSize(gHandle) + sizeof(STqSerializedHead); + int sz = tqGroupSSize(pGroup) + sizeof(STqSerializedHead); if (sz > (*ppHead)->ssize) { void* tmpPtr = realloc(*ppHead, sz); if (tmpPtr == NULL) { @@ -272,53 +349,53 @@ int tqSerializeGroupHandle(const STqGroupHandle* gHandle, STqSerializedHead** pp } void* ptr = (*ppHead)->content; // do serialization - *(int64_t*)ptr = gHandle->cId; + *(int64_t*)ptr = pGroup->clientId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = gHandle->cgId; + *(int64_t*)ptr = pGroup->cgId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int32_t*)ptr = gHandle->topicNum; + *(int32_t*)ptr = pGroup->topicNum; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - if (gHandle->topicNum > 0) { - tqSerializeListHandle(gHandle->head, ptr); + if (pGroup->topicNum > 0) { + tqSerializeListHandle(pGroup->head, ptr); } return 0; } -void* tqSerializeListHandle(STqListHandle* listHandle, void* ptr) { - STqListHandle* node = listHandle; +void* tqSerializeListHandle(STqList* listHandle, void* ptr) { + STqList* node = listHandle; ASSERT(node != NULL); while (node) { - ptr = tqSerializeBufHandle(&node->bufHandle, ptr); + ptr = tqSerializeTopic(&node->topic, ptr); node = node->next; } return ptr; } -void* tqSerializeBufHandle(STqBufferHandle* bufHandle, void* ptr) { - *(int64_t*)ptr = bufHandle->nextConsumeOffset; +void* tqSerializeTopic(STqTopic* pTopic, void* ptr) { + *(int64_t*)ptr = pTopic->nextConsumeOffset; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = bufHandle->topicId; + *(int64_t*)ptr = pTopic->topicId; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int32_t*)ptr = bufHandle->head; + *(int32_t*)ptr = pTopic->head; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - *(int32_t*)ptr = bufHandle->tail; + *(int32_t*)ptr = pTopic->tail; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqSerializeBufItem(&bufHandle->buffer[i], ptr); + ptr = tqSerializeItem(&pTopic->buffer[i], ptr); } return ptr; } -void* tqSerializeBufItem(STqBufferItem* bufItem, void* ptr) { +void* tqSerializeItem(STqMsgItem* bufItem, void* ptr) { // TODO: do we need serialize this? // mainly for executor return ptr; } -const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHandle** ppGHandle) { - STqGroupHandle* gHandle = *ppGHandle; - const void* ptr = pHead->content; - gHandle->cId = *(int64_t*)ptr; +const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup) { + STqGroup* gHandle = *ppGroup; + const void* ptr = pHead->content; + gHandle->clientId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); gHandle->cgId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); @@ -326,63 +403,63 @@ const void* tqDeserializeGroupHandle(const STqSerializedHead* pHead, STqGroupHan gHandle->topicNum = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); gHandle->head = NULL; - STqListHandle* node = gHandle->head; + STqList* node = gHandle->head; for (int i = 0; i < gHandle->topicNum; i++) { if (gHandle->head == NULL) { - if ((node = malloc(sizeof(STqListHandle))) == NULL) { + if ((node = malloc(sizeof(STqList))) == NULL) { // TODO: error return NULL; } node->next = NULL; - ptr = tqDeserializeBufHandle(ptr, &node->bufHandle); + ptr = tqDeserializeTopic(ptr, &node->topic); gHandle->head = node; } else { - node->next = malloc(sizeof(STqListHandle)); + node->next = malloc(sizeof(STqList)); if (node->next == NULL) { // TODO: error return NULL; } node->next->next = NULL; - ptr = tqDeserializeBufHandle(ptr, &node->next->bufHandle); + ptr = tqDeserializeTopic(ptr, &node->next->topic); node = node->next; } } return ptr; } -const void* tqDeserializeBufHandle(const void* pBytes, STqBufferHandle* bufHandle) { +const void* tqDeserializeTopic(const void* pBytes, STqTopic* topic) { const void* ptr = pBytes; - bufHandle->nextConsumeOffset = *(int64_t*)ptr; + topic->nextConsumeOffset = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - bufHandle->topicId = *(int64_t*)ptr; + topic->topicId = *(int64_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - bufHandle->head = *(int32_t*)ptr; + topic->head = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - bufHandle->tail = *(int32_t*)ptr; + topic->tail = *(int32_t*)ptr; ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); for (int i = 0; i < TQ_BUFFER_SIZE; i++) { - ptr = tqDeserializeBufItem(ptr, &bufHandle->buffer[i]); + ptr = tqDeserializeItem(ptr, &topic->buffer[i]); } return ptr; } -const void* tqDeserializeBufItem(const void* pBytes, STqBufferItem* bufItem) { return pBytes; } +const void* tqDeserializeItem(const void* pBytes, STqMsgItem* bufItem) { return pBytes; } // TODO: make this a macro -int tqGetgHandleSSize(const STqGroupHandle* gHandle) { +int tqGroupSSize(const STqGroup* gHandle) { return sizeof(int64_t) * 2 // cId + cgId + sizeof(int32_t) // topicNum - + gHandle->topicNum * tqBufHandleSSize(); + + gHandle->topicNum * tqTopicSSize(); } // TODO: make this a macro -int tqBufHandleSSize() { +int tqTopicSSize() { return sizeof(int64_t) * 2 // nextConsumeOffset + topicId + sizeof(int32_t) * 2 // head + tail - + TQ_BUFFER_SIZE * tqBufItemSSize(); + + TQ_BUFFER_SIZE * tqItemSSize(); } -int tqBufItemSSize() { +int tqItemSSize() { // TODO: do this need serialization? // mainly for executor return 0; diff --git a/source/dnode/vnode/tq/src/tqMetaStore.c b/source/dnode/vnode/tq/src/tqMetaStore.c index 082f0ad28e30356794d9cc12b69eac2668a6fb9c..57e20010e3b12b615875e1ecaba1eddd05161562 100644 --- a/source/dnode/vnode/tq/src/tqMetaStore.c +++ b/source/dnode/vnode/tq/src/tqMetaStore.c @@ -56,6 +56,7 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { int offset = tqSeekLastPage(fd); int nBytes; if ((nBytes = read(fd, pBuf, TQ_PAGE_SIZE)) == -1) { + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } if (nBytes == 0) { @@ -71,7 +72,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial int32_t tqConfigFlag) { STqMetaStore* pMeta = malloc(sizeof(STqMetaStore)); if (pMeta == NULL) { - // close + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } memset(pMeta, 0, sizeof(STqMetaStore)); @@ -79,8 +80,9 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial // concat data file name and index file name size_t pathLen = strlen(path); pMeta->dirPath = malloc(pathLen + 1); - if (pMeta->dirPath != NULL) { - // TODO: memory insufficient + if (pMeta->dirPath == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return NULL; } strcpy(pMeta->dirPath, path); @@ -88,13 +90,14 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial strcpy(name, path); if (taosDirExist(name) != 0 && taosMkDir(name) != 0) { - ASSERT(false); + terrno = TSDB_CODE_TQ_FAILED_TO_CREATE_DIR; + tqError("failed to create dir:%s since %s ", name, terrstr()); } strcat(name, "/" TQ_IDX_NAME); int idxFd = open(name, O_RDWR | O_CREAT, 0755); if (idxFd < 0) { - ASSERT(false); - // close file + terrno = TAOS_SYSTEM_ERROR(errno); + tqError("failed to open file:%s since %s ", name, terrstr()); // free memory return NULL; } @@ -102,9 +105,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial pMeta->idxFd = idxFd; pMeta->unpersistHead = malloc(sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { - ASSERT(false); - // close file - // free memory + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } memset(pMeta->unpersistHead, 0, sizeof(STqMetaList)); @@ -114,7 +115,8 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial strcat(name, "/" TQ_META_NAME); int fileFd = open(name, O_RDWR | O_CREAT, 0755); if (fileFd < 0) { - ASSERT(false); + terrno = TAOS_SYSTEM_ERROR(errno); + tqError("failed to open file:%s since %s", name, terrstr()); return NULL; } @@ -129,7 +131,7 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial STqIdxPageBuf idxBuf; STqSerializedHead* serializedObj = malloc(TQ_PAGE_SIZE); if (serializedObj == NULL) { - // TODO:memory insufficient + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; } int idxRead; int allocated = TQ_PAGE_SIZE; @@ -137,14 +139,16 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial while ((idxRead = read(idxFd, &idxBuf, TQ_PAGE_SIZE))) { if (idxRead == -1) { // TODO: handle error - ASSERT(false); + terrno = TAOS_SYSTEM_ERROR(errno); + tqError("failed to read tq index file since %s", terrstr()); } ASSERT(idxBuf.head.writeOffset == idxRead); // loop read every entry for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { STqMetaList* pNode = malloc(sizeof(STqMetaList)); if (pNode == NULL) { - // TODO: free memory and return error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + // TODO: free memory } memset(pNode, 0, sizeof(STqMetaList)); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); @@ -153,7 +157,8 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial if (allocated < pNode->handle.serializedSize) { void* ptr = realloc(serializedObj, pNode->handle.serializedSize); if (ptr == NULL) { - // TODO: memory insufficient + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + // TODO: free memory } serializedObj = ptr; allocated = pNode->handle.serializedSize; @@ -292,7 +297,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { STqMetaList* pNode = pHead->unpersistNext; STqSerializedHead* pSHead = malloc(sizeof(STqSerializedHead)); if (pSHead == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } pSHead->ver = TQ_SVER; @@ -403,7 +408,6 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu STqMetaList* pNode = pMeta->bucket[bucketKey]; while (pNode) { if (pNode->handle.key == key) { - // TODO: think about thread safety if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInUse); } @@ -416,7 +420,7 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu } STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); if (pNewNode == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } memset(pNewNode, 0, sizeof(STqMetaList)); @@ -470,10 +474,10 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va STqMetaList* pNode = pMeta->bucket[bucketKey]; while (pNode) { if (pNode->handle.key == key) { - // TODO: think about thread safety if (pNode->handle.valueInTxn) { - if (TqDupIntxnReject(pMeta->tqConfigFlag)) { - return -2; + if (tqDupIntxnReject(pMeta->tqConfigFlag)) { + terrno = TSDB_CODE_TQ_META_KEY_DUP_IN_TXN; + return -1; } if (pNode->handle.valueInTxn != TQ_DELETE_TOKEN) { pMeta->pDeleter(pNode->handle.valueInTxn); @@ -488,7 +492,7 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va } STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); if (pNewNode == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } memset(pNewNode, 0, sizeof(STqMetaList)); @@ -505,7 +509,7 @@ int32_t tqHandleMovePut(STqMetaStore* pMeta, int64_t key, void* value) { return int32_t tqHandleCopyPut(STqMetaStore* pMeta, int64_t key, void* value, size_t vsize) { void* vmem = malloc(vsize); if (vmem == NULL) { - // TODO: memory error + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } memcpy(vmem, value, vsize); @@ -535,6 +539,7 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { while (pNode) { if (pNode->handle.key == key) { if (pNode->handle.valueInTxn == NULL) { + terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN; return -1; } if (pNode->handle.valueInUse && pNode->handle.valueInUse != TQ_DELETE_TOKEN) { @@ -548,7 +553,8 @@ int32_t tqHandleCommit(STqMetaStore* pMeta, int64_t key) { pNode = pNode->next; } } - return -2; + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; + return -1; } int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { @@ -564,12 +570,14 @@ int32_t tqHandleAbort(STqMetaStore* pMeta, int64_t key) { tqLinkUnpersist(pMeta, pNode); return 0; } + terrno = TSDB_CODE_TQ_META_KEY_NOT_IN_TXN; return -1; } else { pNode = pNode->next; } } - return -2; + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; + return -1; } int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { @@ -588,7 +596,7 @@ int32_t tqHandleDel(STqMetaStore* pMeta, int64_t key) { pNode = pNode->next; } } - // no such key + terrno = TSDB_CODE_TQ_META_NO_SUCH_KEY; return -1; } diff --git a/source/dnode/vnode/tq/test/tqMetaTest.cpp b/source/dnode/vnode/tq/test/tqMetaTest.cpp index 58263efa71ec77db6a5fbfc973d4f7c9edd55714..d3c9b50e4abd884903f50b2bf5d10bd8ec71eb00 100644 --- a/source/dnode/vnode/tq/test/tqMetaTest.cpp +++ b/source/dnode/vnode/tq/test/tqMetaTest.cpp @@ -10,8 +10,8 @@ struct Foo { }; int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { - Foo* foo = (Foo*) pObj; - if((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { + Foo* foo = (Foo*)pObj; + if ((*ppHead) == NULL || (*ppHead)->ssize < sizeof(STqSerializedHead) + sizeof(int32_t)) { *ppHead = (STqSerializedHead*)realloc(*ppHead, sizeof(STqSerializedHead) + sizeof(int32_t)); (*ppHead)->ssize = sizeof(STqSerializedHead) + sizeof(int32_t); } @@ -20,36 +20,28 @@ int FooSerializer(const void* pObj, STqSerializedHead** ppHead) { } const void* FooDeserializer(const STqSerializedHead* pHead, void** ppObj) { - if(*ppObj == NULL) { + if (*ppObj == NULL) { *ppObj = realloc(*ppObj, sizeof(int32_t)); } Foo* pFoo = *(Foo**)ppObj; - pFoo->a = *(int32_t*)pHead->content; + pFoo->a = *(int32_t*)pHead->content; return NULL; } -void FooDeleter(void* pObj) { - free(pObj); -} +void FooDeleter(void* pObj) { free(pObj); } class TqMetaUpdateAppendTest : public ::testing::Test { - protected: - - void SetUp() override { - taosRemoveDir(pathName); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); - ASSERT(pMeta); - } - - void TearDown() override { - tqStoreClose(pMeta); - } - - TqMetaStore* pMeta; - const char* pathName = "/tmp/tq_test"; + protected: + void SetUp() override { + taosRemoveDir(pathName); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); + ASSERT(pMeta); + } + + void TearDown() override { tqStoreClose(pMeta); } + + STqMetaStore* pMeta; + const char* pathName = "/tmp/tq_test"; }; TEST_F(TqMetaUpdateAppendTest, copyPutTest) { @@ -57,11 +49,11 @@ TEST_F(TqMetaUpdateAppendTest, copyPutTest) { foo.a = 3; tqHandleCopyPut(pMeta, 1, &foo, sizeof(Foo)); - Foo* pFoo = (Foo*) tqHandleGet(pMeta, 1); + Foo* pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqHandleCommit(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo->a, 3); } @@ -78,10 +70,7 @@ TEST_F(TqMetaUpdateAppendTest, persistTest) { EXPECT_EQ(pBar == NULL, true); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); pBar = (Foo*)tqHandleGet(pMeta, 1); @@ -97,7 +86,7 @@ TEST_F(TqMetaUpdateAppendTest, uncommittedTest) { pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); } @@ -106,11 +95,11 @@ TEST_F(TqMetaUpdateAppendTest, abortTest) { pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqHandleAbort(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); } @@ -119,32 +108,29 @@ TEST_F(TqMetaUpdateAppendTest, deleteTest) { pFoo->a = 3; tqHandleMovePut(pMeta, 1, pFoo); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqHandleCommit(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); ASSERT_EQ(pFoo != NULL, true); EXPECT_EQ(pFoo->a, 3); tqHandleDel(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); ASSERT_EQ(pFoo != NULL, true); EXPECT_EQ(pFoo->a, 3); tqHandleCommit(pMeta, 1); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); - pFoo = (Foo*) tqHandleGet(pMeta, 1); + pFoo = (Foo*)tqHandleGet(pMeta, 1); EXPECT_EQ(pFoo == NULL, true); } @@ -162,10 +148,7 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) { EXPECT_EQ(pFoo1->a, 3); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); pFoo1 = (Foo*)tqHandleGet(pMeta, 1); @@ -177,10 +160,7 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) { EXPECT_EQ(pFoo1->a, 4); tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); pFoo1 = (Foo*)tqHandleGet(pMeta, 1); @@ -190,13 +170,13 @@ TEST_F(TqMetaUpdateAppendTest, intxnPersist) { TEST_F(TqMetaUpdateAppendTest, multiplePage) { srand(0); std::vector v; - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { v.push_back(rand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 500; i++) { + for (int i = 0; i < 500; i++) { tqHandleCommit(pMeta, i); Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; @@ -204,38 +184,34 @@ TEST_F(TqMetaUpdateAppendTest, multiplePage) { } tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); - - for(int i = 500; i < 1000; i++) { + + for (int i = 500; i < 1000; i++) { tqHandleCommit(pMeta, i); Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - } TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { srand(0); std::vector v; - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { v.push_back(rand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 500; i++) { + for (int i = 0; i < 500; i++) { tqHandleCommit(pMeta, i); v[i] = rand(); Foo foo; @@ -243,25 +219,22 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 500; i < 1000; i++) { + for (int i = 500; i < 1000; i++) { v[i] = rand(); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { tqHandleCommit(pMeta, i); } tqStoreClose(pMeta); - pMeta = tqStoreOpen(pathName, - FooSerializer, FooDeserializer, FooDeleter, - TQ_UPDATE_APPEND - ); + pMeta = tqStoreOpen(pathName, FooSerializer, FooDeserializer, FooDeleter, TQ_UPDATE_APPEND); ASSERT(pMeta); - - for(int i = 500; i < 1000; i++) { + + for (int i = 500; i < 1000; i++) { v[i] = rand(); Foo foo; foo.a = v[i]; @@ -269,40 +242,38 @@ TEST_F(TqMetaUpdateAppendTest, multipleRewrite) { tqHandleCommit(pMeta, i); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - } TEST_F(TqMetaUpdateAppendTest, dupCommit) { srand(0); std::vector v; - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { v.push_back(rand()); Foo foo; foo.a = v[i]; tqHandleCopyPut(pMeta, i, &foo, sizeof(Foo)); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { int ret = tqHandleCommit(pMeta, i); EXPECT_EQ(ret, 0); ret = tqHandleCommit(pMeta, i); EXPECT_EQ(ret, -1); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { int ret = tqHandleCommit(pMeta, i); EXPECT_EQ(ret, -1); } - for(int i = 0; i < 1000; i++) { + for (int i = 0; i < 1000; i++) { Foo* pFoo = (Foo*)tqHandleGet(pMeta, i); ASSERT_EQ(pFoo != NULL, true) << " at idx " << i << "\n"; EXPECT_EQ(pFoo->a, v[i]); } - } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index e14c58d4122f1f830bcac17c2c358ef6632ef15c..29c80ae9ec05453dd23605c39d43fa562c2aa2d9 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -61,8 +61,8 @@ void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { pReq->daysToKeep0 = htonl(3650); pReq->daysToKeep1 = htonl(3650); pReq->daysToKeep2 = htonl(3650); - pReq->minRowsPerFileBlock = htonl(100); - pReq->maxRowsPerFileBlock = htonl(4096); + pReq->minRows = htonl(100); + pReq->maxRows = htonl(4096); pReq->commitTime = htonl(3600); pReq->fsyncPeriod = htonl(3000); pReq->walLevel = 1; diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index a258cb834dcbe4a15845c690b706d46b09674c78..22bb9d1d0f2ebbf342b93ef6f40347e7437685f1 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -76,7 +76,7 @@ struct SIndexMultiTermQuery { // field and key; typedef struct SIndexTerm { - int64_t suid; + int64_t suid; SIndexOperOnColumn operType; // oper type, add/del/update uint8_t colType; // term data type, str/interger/json char *colName; diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index ea090389bba1ff1a12b91ebcac2d15827916286a..ac9a59fa042775fa101e9fec1db54fe8fb995200 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -34,8 +34,14 @@ typedef struct WriterCtx { int (*flush)(struct WriterCtx *ctx); WriterType type; union { - int fd; - void *mem; + struct { + int fd; + bool readOnly; + } file; + struct { + int32_t capa; + char *buf; + } mem; }; int32_t offset; int32_t limit; @@ -45,7 +51,7 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); static int writeCtxDoFlush(WriterCtx *ctx); -WriterCtx* writerCtxCreate(WriterType type, bool readOnly); +WriterCtx* writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity); void writerCtxDestroy(WriterCtx *w); typedef uint32_t CheckSummer; @@ -66,7 +72,7 @@ int fstCountingWriterFlush(FstCountingWriter *write); uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write); -FstCountingWriter *fstCountingWriterCreate(void *wtr, bool readOnly); +FstCountingWriter *fstCountingWriterCreate(void *wtr); void fstCountingWriterDestroy(FstCountingWriter *w); diff --git a/source/libs/index/inc/index_tfile.h b/source/libs/index/inc/index_tfile.h index c3f4bd25e5e6a792120f1ac8880bfbc57915dfef..979c0b06397bb9d928d6c1b48294eadd5bee4246 100644 --- a/source/libs/index/inc/index_tfile.h +++ b/source/libs/index/inc/index_tfile.h @@ -18,23 +18,81 @@ #include "index.h" #include "indexInt.h" #include "tlockfree.h" -#include "tskiplist.h" +#include "index_tfile.h" +#include "index_fst_counting_writer.h" +#include "index_fst.h" #ifdef __cplusplus extern "C" { #endif -typedef struct IndexTFile { + + +typedef struct TFileCacheKey { + uint64_t suid; + uint8_t colType; + int32_t version; + const char *colName; + int32_t nColName; +} TFileCacheKey; + + +// table cache +// refactor to LRU cache later +typedef struct TFileCache { + SHashObj *tableCache; + int16_t capacity; + // add more param +} TFileCache; + + +typedef struct TFileWriter { + FstBuilder *fb; + WriterCtx *wc; +} TFileWriter; + +typedef struct TFileReader { T_REF_DECLARE() + Fst *fst; + +} TFileReader; + +typedef struct IndexTFile { + char *path; + TFileReader *tb; + TFileWriter *tw; } IndexTFile; +typedef struct TFileWriterOpt { + uint64_t suid; + int8_t colType; + char *colName; + int32_t nColName; + int32_t version; +} TFileWriterOpt; +typedef struct TFileReaderOpt { + uint64_t suid; + char *colName; + int32_t nColName; + +} TFileReaderOpt; + +// tfile cache +TFileCache *tfileCacheCreate(); +void tfileCacheDestroy(TFileCache *tcache); +TFileReader* tfileCacheGet(TFileCache *tcache, TFileCacheKey *key); +void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader); + +TFileWriter *tfileWriterCreate(const char *suid, const char *colName); IndexTFile *indexTFileCreate(); +int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid); int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result); + #ifdef __cplusplus } diff --git a/source/libs/index/inc/index_util.h b/source/libs/index/inc/index_util.h new file mode 100644 index 0000000000000000000000000000000000000000..4ab517ecfafe6927dde70bbbd6fb0b7076e2b5f9 --- /dev/null +++ b/source/libs/index/inc/index_util.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef __INDEX_UTIL_H__ +#define __INDEX_UTIL_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +#define SERIALIZE_MEM_TO_BUF(buf, key, mem) \ + do { \ + memcpy((void *)buf, (void *)(&key->mem), sizeof(key->mem)); \ + buf += sizeof(key->mem); \ + } while (0) + +#define SERIALIZE_STR_MEM_TO_BUF(buf, key, mem, len) \ + do { \ + memcpy((void *)buf, (void *)key->mem, len); \ + buf += len; \ + } while (0) + +#define SERIALIZE_VAR_TO_BUF(buf, var, type) \ + do { \ + type c = var; \ + assert(sizeof(var) == sizeof(type));\ + memcpy((void *)buf, (void *)&c, sizeof(c)); \ + buf += sizeof(c); \ + } while (0) + +#define SERIALIZE_STR_VAR_TO_BUF(buf, var, len) \ + do { \ + memcpy((void *)buf, (void *)var, len); \ + buf += len;\ + } while (0) + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/libs/index/src/index_cache.c b/source/libs/index/src/index_cache.c index 3c52275a4c96f2f0f12f6fae012c9681f2a37c99..5813c99164ee11f4a196b1d69638ef8ad37847b2 100644 --- a/source/libs/index/src/index_cache.c +++ b/source/libs/index/src/index_cache.c @@ -15,6 +15,7 @@ #include "index_cache.h" #include "tcompare.h" +#include "index_util.h" #define MAX_INDEX_KEY_LEN 256// test only, change later @@ -110,35 +111,22 @@ int indexCachePut(void *cache, SIndexTerm *term, int16_t colId, int32_t version, if (cache == NULL) { return -1;} IndexCache *pCache = cache; - // encode data int32_t total = CACHE_KEY_LEN(term); - char *buf = calloc(1, total); char *p = buf; - memcpy(p, &total, sizeof(total)); - p += sizeof(total); - - memcpy(p, &colId, sizeof(colId)); - p += sizeof(colId); + SERIALIZE_VAR_TO_BUF(p, total,int32_t); + SERIALIZE_VAR_TO_BUF(p, colId, int16_t); - memcpy(p, &term->colType, sizeof(term->colType)); - p += sizeof(term->colType); + SERIALIZE_MEM_TO_BUF(p, term, colType); + SERIALIZE_MEM_TO_BUF(p, term, nColVal); + SERIALIZE_STR_MEM_TO_BUF(p, term, colVal, term->nColVal); - memcpy(p, &term->nColVal, sizeof(term->nColVal)); - p += sizeof(term->nColVal); - memcpy(p, term->colVal, term->nColVal); - p += term->nColVal; - - memcpy(p, &version, sizeof(version)); - p += sizeof(version); - - memcpy(p, &uid, sizeof(uid)); - p += sizeof(uid); + SERIALIZE_VAR_TO_BUF(p, version, int32_t); + SERIALIZE_VAR_TO_BUF(p, uid, uint64_t); - memcpy(p, &term->operType, sizeof(term->operType)); - p += sizeof(term->operType); + SERIALIZE_MEM_TO_BUF(p, term, operType); tSkipListPut(pCache->skiplist, (void *)buf); return 0; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 7aaa498864fff74dccccdb2446c190b24272fcb2..0f00aacf3bd0fcd746042977d4f0b62ad3d3ba8f 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -779,7 +779,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty) { if (NULL == b) { return b; } - b->wrt = fstCountingWriterCreate(w, false); + b->wrt = fstCountingWriterCreate(w); b->unfinished = fstUnFinishedNodesCreate(); b->registry = fstRegistryCreate(10000, 2) ; b->last = fstSliceCreate(NULL, 0); diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 9ec346cebc026cba7d03e1cf39890a6c262d1a9a..3497b9703d2d875d2d093e3fb4fc2ec7c22e32b0 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -23,9 +23,9 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { } if (ctx->type == TFile) { - assert(len == tfWrite(ctx->fd, buf, len)); + assert(len == tfWrite(ctx->file.fd, buf, len)); } else { - memcpy(ctx->mem + ctx->offset, buf, len); + memcpy(ctx->mem.buf+ ctx->offset, buf, len); } ctx->offset += len; return len; @@ -33,9 +33,9 @@ static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { int nRead = 0; if (ctx->type == TFile) { - nRead = tfRead(ctx->fd, buf, len); + nRead = tfRead(ctx->file.fd, buf, len); } else { - memcpy(buf, ctx->mem + ctx->offset, len); + memcpy(buf, ctx->mem.buf + ctx->offset, len); } ctx->offset += nRead; @@ -44,63 +44,64 @@ static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { static int writeCtxDoFlush(WriterCtx *ctx) { if (ctx->type == TFile) { //tfFsync(ctx->fd); - //tfFlush(ctx->fd); + //tfFlush(ctx->file.fd); } else { // do nothing } return 1; } -WriterCtx* writerCtxCreate(WriterType type, bool readOnly) { +WriterCtx* writerCtxCreate(WriterType type, const char *path, bool readOnly, int32_t capacity) { WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); if (ctx == NULL) { return NULL; } ctx->type = type; if (ctx->type == TFile) { - tfInit(); // ugly code, refactor later + ctx->file.readOnly = readOnly; if (readOnly == false) { - ctx->fd = tfOpenCreateWriteAppend(tmpFile); + ctx->file.fd = tfOpenCreateWriteAppend(tmpFile); } else { - ctx->fd = tfOpenReadWrite(tmpFile); + ctx->file.fd = tfOpenReadWrite(tmpFile); } - if (ctx->fd < 0) { + if (ctx->file.fd < 0) { indexError("open file error %d", errno); } } else if (ctx->type == TMemory) { - ctx->mem = calloc(1, DefaultMem * sizeof(uint8_t)); + ctx->mem.buf = calloc(1, sizeof(char) * capacity); + ctx->mem.capa = capacity; } ctx->write = writeCtxDoWrite; ctx->read = writeCtxDoRead; ctx->flush = writeCtxDoFlush; ctx->offset = 0; - ctx->limit = DefaultMem; + ctx->limit = capacity; return ctx; } void writerCtxDestroy(WriterCtx *ctx) { if (ctx->type == TMemory) { - free(ctx->mem); + free(ctx->mem.buf); } else { - tfClose(ctx->fd); - tfCleanup(); + tfClose(ctx->file.fd); } free(ctx); } -FstCountingWriter *fstCountingWriterCreate(void *wrt, bool readOnly) { +FstCountingWriter *fstCountingWriterCreate(void *wrt) { FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); if (cw == NULL) { return NULL; } - cw->wrt = (void *)(writerCtxCreate(TFile, readOnly)); + cw->wrt = wrt; + //(void *)(writerCtxCreate(TFile, readOnly)); return cw; } void fstCountingWriterDestroy(FstCountingWriter *cw) { // free wrt object: close fd or free mem fstCountingWriterFlush(cw); - writerCtxDestroy((WriterCtx *)(cw->wrt)); + //writerCtxDestroy((WriterCtx *)(cw->wrt)); free(cw); } @@ -124,6 +125,7 @@ int fstCountingWriterRead(FstCountingWriter *write, uint8_t *buf, uint32_t len) } uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { + return 0; } int fstCountingWriterFlush(FstCountingWriter *write) { diff --git a/source/libs/index/src/index_tfile.c b/source/libs/index/src/index_tfile.c index a1bba56391ce445899fe1fc23b8e422c8cf3cd8f..81a7f9f44351bff3ea77146f5182c3386bdf6dd9 100644 --- a/source/libs/index/src/index_tfile.c +++ b/source/libs/index/src/index_tfile.c @@ -14,6 +14,50 @@ */ #include "index_tfile.h" +#include "index_fst.h" +#include "index_util.h" + + + +static void tfileSerialCacheKey(TFileCacheKey *key, char *buf) { + SERIALIZE_MEM_TO_BUF(buf, key, suid); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_MEM_TO_BUF(buf, key, colType); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_MEM_TO_BUF(buf, key, version); + SERIALIZE_VAR_TO_BUF(buf, '_', char); + SERIALIZE_STR_MEM_TO_BUF(buf, key, colName, key->nColName); +} + +TFileCache *tfileCacheCreate() { + TFileCache *tcache = calloc(1, sizeof(TFileCache)); + if (tcache == NULL) { return NULL; } + + tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + tcache->capacity = 64; + return tcache; +} +void tfileCacheDestroy(TFileCache *tcache) { + + free(tcache); + +} + +TFileReader *tfileCacheGet(TFileCache *tcache, TFileCacheKey *key) { + char buf[128] = {0}; + tfileSerialCacheKey(key, buf); + TFileReader *reader = taosHashGet(tcache->tableCache, buf, strlen(buf)); + return reader; +} +void tfileCachePut(TFileCache *tcache, TFileCacheKey *key, TFileReader *reader) { + char buf[128] = {0}; + tfileSerialCacheKey(key, buf); + taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void *)); + return; +} + + + IndexTFile *indexTFileCreate() { IndexTFile *tfile = calloc(1, sizeof(IndexTFile)); @@ -22,10 +66,24 @@ IndexTFile *indexTFileCreate() { void IndexTFileDestroy(IndexTFile *tfile) { free(tfile); } + + int indexTFileSearch(void *tfile, SIndexTermQuery *query, SArray *result) { IndexTFile *ptfile = (IndexTFile *)tfile; + return 0; } +int indexTFilePut(void *tfile, SIndexTerm *term, uint64_t uid) { + TFileWriterOpt wOpt = {.suid = term->suid, + .colType = term->colType, + .colName = term->colName, + .nColName= term->nColName, + .version = 1}; + + + + return 0; +} diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 9dff2e9ea0ac901390cde63feee01f50c3179fe6..9baabb9610c6bea3c04469538891df32c439a840 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -45,7 +45,7 @@ class FstWriter { class FstReadMemory { public: FstReadMemory(size_t size) { - _w = fstCountingWriterCreate(NULL, true); + _w = fstCountingWriterCreate(NULL); _size = size; memset((void *)&_s, 0, sizeof(_s)); } diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index de7cdd58b85227c3a4dbb1742a0789fe01413802..32906f7800de1e724bc2ee1e43511f051ab93eeb 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -4,7 +4,8 @@ #include "parserInt.h" #include "taosmsg.h" -SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen); +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); +SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SShowMsg* buildShowMsg(SShowInfo* pShowInfo, int64_t id, char* msgBuf, int32_t msgLen); SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, char* msgBuf, int32_t msgLen); diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 1b46faececee5dabb589d8fdae2bccc0ddf0aca1..6a752cc195462479f6d480d3f5856bb4e9ff7854 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -1,7 +1,7 @@ #include "parserInt.h" #include "parserUtil.h" -SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgBuf, int32_t msgLen) { +SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen) { SCreateUserMsg* pMsg = (SCreateUserMsg*)calloc(1, sizeof(SCreateUserMsg)); if (pMsg == NULL) { // tscError("0x%" PRIx64 " failed to malloc for query msg", id); @@ -20,6 +20,24 @@ SCreateUserMsg* buildUserManipulationMsg(SSqlInfo* pInfo, int64_t id, char* msgB strncpy(pMsg->pass, pUser->passwd.z, pUser->passwd.n); } + *outputLen = sizeof(SUserInfo); + return pMsg; +} + +SDropUserMsg* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, char* msgBuf, int32_t msgBufLen) { + SToken* pName = taosArrayGet(pInfo->pMiscInfo->a, 0); + if (pName->n >= TSDB_USER_LEN) { + return NULL; + } + + + SDropUserMsg* pMsg = calloc(1, sizeof(SDropUserMsg)); + if (pMsg == NULL) { + return NULL; + } + + strncpy(pMsg->user, pName->z, pName->n); + *msgLen = sizeof(SDropUserMsg); return pMsg; } @@ -89,7 +107,7 @@ static int32_t setTimePrecision(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreate pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - SToken* pToken = &pCreateDbInfo->precision; + SToken* pToken = (SToken*) &pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); @@ -116,8 +134,8 @@ static void doSetDbOptions(SCreateDbMsg* pMsg, const SCreateDbInfo* pCreateDb) { pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime); - pMsg->minRowsPerFileBlock = htonl(pCreateDb->minRowsPerBlock); - pMsg->maxRowsPerFileBlock = htonl(pCreateDb->maxRowsPerBlock); + pMsg->minRows = htonl(pCreateDb->minRowsPerBlock); + pMsg->maxRows = htonl(pCreateDb->maxRowsPerBlock); pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); pMsg->compression = pCreateDb->compressionLevel; pMsg->walLevel = (char)pCreateDb->walLevel; @@ -141,7 +159,6 @@ int32_t setDbOptions(SCreateDbMsg* pCreateDbMsg, const SCreateDbInfo* pCreateDbS // todo configurable pCreateDbMsg->numOfVgroups = htonl(2); - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index 135774cd3b6921130c183efabf1dcacc12c51120..ba96c4a79638aaa5bdf26c68f46dc715daf181e3 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -4224,7 +4224,13 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, int64_t id, void** output, in } } - *output = buildUserManipulationMsg(pInfo, id, msgBuf, msgBufLen); + *output = buildUserManipulationMsg(pInfo, outputLen, id, msgBuf, msgBufLen); + break; + } + + case TSDB_SQL_DROP_ACCT: + case TSDB_SQL_DROP_USER: { + *output = buildDropUserMsg(pInfo, outputLen, id, msgBuf, msgBufLen); break; } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index c548f1556a64daeedb9c99033245404f594297ce..502946675b40738c8ec7d3e35287669ce04dab71 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -1,10 +1,7 @@ +#include "os.h" #include "queryInfoUtil.h" -#include -#include "astGenerator.h" #include "function.h" -#include "os.h" #include "parser.h" -#include "parserInt.h" #include "parserUtil.h" static struct SSchema _s = { diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 9bc029cbd8d23906872f954aad998caff6262c68..e7271c3cb2a2901f339500369030d3cb95ccf9e4 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -38,7 +38,7 @@ public: virtual TableBuilder& setVgid(int16_t vgid) { schema()->vgId = vgid; - meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + meta_->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); return *this; } @@ -118,9 +118,9 @@ public: meta_[db][tbname].reset(new MockTableMeta()); meta_[db][tbname]->schema.reset(table.release()); meta_[db][tbname]->schema->uid = id_++; - meta_[db][tbname]->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + meta_[db][tbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); // super table - meta_[db][stbname]->vgs.emplace_back(SVgroupInfo{.vgId = vgid, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); + meta_[db][stbname]->vgs.emplace_back((SVgroupInfo){.vgId = vgid, .hashBegin = 0, .hashEnd = 0, .inUse = 0, .numOfEps = 3, .epAddr = {{"dnode_1", 6030}, {"dnode_2", 6030}, {"dnode_3", 6030}}}); } void showTables() const { diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 9ca8b6789b328dc00f8d23e1c35264b70ece8413..728c47af1e71e4c8b15d1f2a921c34149f3c098f 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -294,7 +294,7 @@ void msgInit() { tscBuildMsg[TSDB_SQL_DROP_DNODE] = tscBuildDropDnodeMsg; tscBuildMsg[TSDB_SQL_CFG_DNODE] = tscBuildCfgDnodeMsg; tscBuildMsg[TSDB_SQL_ALTER_TABLE] = tscBuildAlterTableMsg; - tscBuildMsg[TSDB_SQL_UPDATE_TAGS_VAL] = tscBuildUpdateTagMsg; + tscBuildMsg[TSDB_SQL_UPDATE_TAG_VAL] = tscBuildUpdateTagMsg; tscBuildMsg[TSDB_SQL_ALTER_DB] = tscAlterDbMsg; tscBuildMsg[TSDB_SQL_COMPACT_VNODE] = tscBuildCompactMsg; diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index cdb4e08205b66abaa91edade1c82928b3e7ce32f..6baaab1ef4b5ce436306ebc98057b2c1e28f599d 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -11,3 +11,5 @@ target_link_libraries( scheduler PRIVATE os util planner qcom common catalog transport ) + +ADD_SUBDIRECTORY(test) \ No newline at end of file diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index db75fc4fdd49336b818b9b57cedf3d0f772817c0..73e7c4d24e1c2693d174dcb7acad4112939778c3 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -73,9 +73,8 @@ typedef struct SQueryJob { SQueryProfileSummary summary; SEpSet dataSrcEps; SEpAddr resEp; - struct SCatalog *catalog; - void *rpc; - SEpSet *mgmtEpSet; + void *transport; + SArray *qnodeList; tsem_t rspSem; int32_t userFetch; int32_t remoteFetch; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 2327fc5b0464bf07add7e3086a362d650cdb8ed6..6014ff9ab6282248f52bfe1aba0535e5b4d4cb71 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -58,8 +58,8 @@ int32_t schBuildTaskRalation(SQueryJob *job, SHashObj *planToTask) { for (int32_t m = 0; m < level->taskNum; ++m) { SQueryTask *task = taosArrayGet(level->subTasks, m); SSubplan *plan = task->plan; - int32_t childNum = (int32_t)taosArrayGetSize(plan->pChildern); - int32_t parentNum = (int32_t)taosArrayGetSize(plan->pParents); + int32_t childNum = plan->pChildern ? (int32_t)taosArrayGetSize(plan->pChildern) : 0; + int32_t parentNum = plan->pParents ? (int32_t)taosArrayGetSize(plan->pParents) : 0; if (childNum > 0) { task->children = taosArrayInit(childNum, POINTER_BYTES); @@ -187,13 +187,19 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SQueryJob *job) { for (int32_t n = 0; n < levelPlanNum; ++n) { SSubplan *plan = taosArrayGet(levelPlans, n); - SQueryTask *task = taosArrayGet(level.subTasks, n); + SQueryTask task = {0}; - task->taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); - task->plan = plan; - task->status = SCH_STATUS_NOT_START; + task.taskId = atomic_add_fetch_64(&schMgmt.taskId, 1); + task.plan = plan; + task.status = SCH_STATUS_NOT_START; - if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &task, POINTER_BYTES)) { + void *p = taosArrayPush(level.subTasks, &task); + if (NULL == p) { + qError("taosArrayPush failed"); + SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &p, POINTER_BYTES)) { qError("taosHashPut failed"); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -225,20 +231,27 @@ _return: SCH_RET(code); } -int32_t schAvailableEpSet(SQueryJob *job, SEpSet *epSet) { +int32_t schSetTaskExecEpSet(SQueryJob *job, SEpSet *epSet) { if (epSet->numOfEps >= SCH_MAX_CONDIDATE_EP_NUM) { return TSDB_CODE_SUCCESS; } - if (SCH_HAS_QNODE_IN_CLUSTER(schMgmt.cfg.clusterType)) { - SCH_ERR_RET(catalogGetQnodeList(job->catalog, job->rpc, job->mgmtEpSet, epSet)); - } else { - for (int32_t i = 0; i < job->dataSrcEps.numOfEps; ++i) { - strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); - epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; - - ++epSet->numOfEps; - } + int32_t qnodeNum = taosArrayGetSize(job->qnodeList); + + for (int32_t i = 0; i < qnodeNum && epSet->numOfEps < tListLen(epSet->port); ++i) { + SEpAddr *addr = taosArrayGet(job->qnodeList, i); + + strncpy(epSet->fqdn[epSet->numOfEps], addr->fqdn, sizeof(addr->fqdn)); + epSet->port[epSet->numOfEps] = addr->port; + + ++epSet->numOfEps; + } + + for (int32_t i = 0; i < job->dataSrcEps.numOfEps && epSet->numOfEps < tListLen(epSet->port); ++i) { + strncpy(epSet->fqdn[epSet->numOfEps], job->dataSrcEps.fqdn[i], sizeof(job->dataSrcEps.fqdn[i])); + epSet->port[epSet->numOfEps] = job->dataSrcEps.port[i]; + + ++epSet->numOfEps; } return TSDB_CODE_SUCCESS; @@ -509,7 +522,12 @@ int32_t schLaunchTask(SQueryJob *job, SQueryTask *task) { SCH_ERR_RET(qSubPlanToString(plan, &task->msg)); if (plan->execEpSet.numOfEps <= 0) { - SCH_ERR_RET(schAvailableEpSet(job, &plan->execEpSet)); + SCH_ERR_RET(schSetTaskExecEpSet(job, &plan->execEpSet)); + } + + if (plan->execEpSet.numOfEps <= 0) { + SCH_TASK_ERR_LOG("invalid execEpSet num:%d", plan->execEpSet.numOfEps); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } SCH_ERR_RET(schAsyncSendMsg(job, task, TSDB_MSG_TYPE_QUERY)); @@ -548,20 +566,23 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { } -int32_t scheduleQueryJob(struct SCatalog *pCatalog, void *pRpc, const SEpSet* pMgmtEps, SQueryDag* pDag, void** pJob) { - if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { +int32_t scheduleExecJob(void *transport, SArray *qnodeList, SQueryDag* pDag, void** pJob) { + if (NULL == transport || NULL == transport ||NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + if (taosArrayGetSize(qnodeList) <= 0) { + qInfo("qnodeList is empty"); + } + int32_t code = 0; SQueryJob *job = calloc(1, sizeof(SQueryJob)); if (NULL == job) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - job->catalog = pCatalog; - job->rpc = pRpc; - job->mgmtEpSet = (SEpSet *)pMgmtEps; + job->transport = transport; + job->qnodeList = qnodeList; SCH_ERR_JRET(schValidateAndBuildJob(pDag, job)); diff --git a/source/libs/scheduler/test/CMakeLists.txt b/source/libs/scheduler/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..00a6d08e5d7be527005a64f0e4111d9a3e35974d --- /dev/null +++ b/source/libs/scheduler/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build scheduler unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(schedulerTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + schedulerTest + PUBLIC os util common catalog transport gtest qcom taos planner scheduler +) + +TARGET_INCLUDE_DIRECTORIES( + schedulerTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scheduler/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/scheduler/inc" +) diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9e94553058e923086808cdb462d27ffa95e9a96e 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "catalog.h" +#include "scheduler.h" +#include "tep.h" +#include "trpc.h" + +namespace { +void mockBuildDag(SQueryDag *dag) { + uint64_t qId = 0x111111111111; + + dag->queryId = qId; + dag->numOfSubplans = 2; + dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES); + SArray *scan = taosArrayInit(1, sizeof(SSubplan)); + SArray *merge = taosArrayInit(1, sizeof(SSubplan)); + + SSubplan scanPlan = {0}; + SSubplan mergePlan = {0}; + + scanPlan.id.queryId = qId; + scanPlan.id.templateId = 0x2222222222; + scanPlan.id.subplanId = 0x3333333333; + scanPlan.type = QUERY_TYPE_SCAN; + scanPlan.level = 1; + scanPlan.execEpSet.numOfEps = 1; + scanPlan.pChildern = NULL; + scanPlan.pParents = taosArrayInit(1, POINTER_BYTES); + + mergePlan.id.queryId = qId; + mergePlan.id.templateId = 0x4444444444; + mergePlan.id.subplanId = 0x5555555555; + mergePlan.type = QUERY_TYPE_MERGE; + mergePlan.level = 0; + mergePlan.execEpSet.numOfEps = 1; + mergePlan.pChildern = taosArrayInit(1, POINTER_BYTES); + mergePlan.pParents = NULL; + + SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); + SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); + + taosArrayPush(mergePointer->pChildern, &scanPointer); + taosArrayPush(scanPointer->pParents, &mergePointer); + + taosArrayPush(dag->pSubplans, &merge); + taosArrayPush(dag->pSubplans, &scan); +} + +} + +TEST(testCase, normalCase) { + void *mockPointer = (void *)0x1; + char *clusterId = "cluster1"; + char *dbname = "1.db1"; + char *tablename = "table1"; + SVgroupInfo vgInfo = {0}; + void *pJob = NULL; + SQueryDag dag = {0}; + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); + + int32_t code = schedulerInit(NULL); + ASSERT_EQ(code, 0); + + mockBuildDag(&dag); + + code = scheduleExecJob(mockPointer, qnodeList, &dag, &pJob); + ASSERT_EQ(code, 0); +} + + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + + + diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 23a923a315765778690c5ba72400148d99527792..5bac59f913462efd0118852d6f39fb047b3713e2 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -14,21 +14,21 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "ulog.h" #include "tlog.h" +#include "os.h" #include "tnote.h" #include "tutil.h" +#include "ulog.h" #include "zlib.h" - -#define MAX_LOGLINE_SIZE (1000) -#define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) -#define MAX_LOGLINE_CONTENT_SIZE (MAX_LOGLINE_SIZE - 100) -#define MAX_LOGLINE_DUMP_SIZE (65 * 1024) -#define MAX_LOGLINE_DUMP_BUFFER_SIZE (MAX_LOGLINE_DUMP_SIZE + 10) + +#define MAX_LOGLINE_SIZE (1000) +#define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) +#define MAX_LOGLINE_CONTENT_SIZE (MAX_LOGLINE_SIZE - 100) +#define MAX_LOGLINE_DUMP_SIZE (65 * 1024) +#define MAX_LOGLINE_DUMP_BUFFER_SIZE (MAX_LOGLINE_DUMP_SIZE + 10) #define MAX_LOGLINE_DUMP_CONTENT_SIZE (MAX_LOGLINE_DUMP_SIZE - 100) -#define LOG_FILE_NAME_LEN 300 +#define LOG_FILE_NAME_LEN 300 #define TSDB_DEFAULT_LOG_BUF_SIZE (20 * 1024 * 1024) // 20MB #define DEFAULT_LOG_INTERVAL 25 @@ -38,13 +38,13 @@ #define LOG_MAX_WAIT_MSEC 1000 #define LOG_BUF_BUFFER(x) ((x)->buffer) -#define LOG_BUF_START(x) ((x)->buffStart) -#define LOG_BUF_END(x) ((x)->buffEnd) -#define LOG_BUF_SIZE(x) ((x)->buffSize) -#define LOG_BUF_MUTEX(x) ((x)->buffMutex) +#define LOG_BUF_START(x) ((x)->buffStart) +#define LOG_BUF_END(x) ((x)->buffEnd) +#define LOG_BUF_SIZE(x) ((x)->buffSize) +#define LOG_BUF_MUTEX(x) ((x)->buffMutex) typedef struct { - char * buffer; + char *buffer; int32_t buffStart; int32_t buffEnd; int32_t buffSize; @@ -57,18 +57,18 @@ typedef struct { } SLogBuff; typedef struct { - int32_t fileNum; - int32_t maxLines; - int32_t lines; - int32_t flag; - int32_t openInProgress; - pid_t pid; - char logName[LOG_FILE_NAME_LEN]; - SLogBuff * logHandle; + int32_t fileNum; + int32_t maxLines; + int32_t lines; + int32_t flag; + int32_t openInProgress; + pid_t pid; + char logName[LOG_FILE_NAME_LEN]; + SLogBuff *logHandle; pthread_mutex_t logMutex; } SLogObj; -int8_t tscEmbeddedInUtil = 0; +int8_t tscEmbeddedInUtil = 0; int32_t tsLogKeepDays = 0; int8_t tsAsyncLog = 1; @@ -93,19 +93,19 @@ int32_t debugFlag = 0; int32_t sDebugFlag = 135; int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; +int32_t tqDebugFlag = 131; int32_t cqDebugFlag = 131; int32_t fsDebugFlag = 135; int32_t ctgDebugFlag = 131; - int64_t dbgEmptyW = 0; int64_t dbgWN = 0; int64_t dbgSmallWN = 0; int64_t dbgBigWN = 0; int64_t dbgWSize = 0; -static SLogObj tsLogObj = { .fileNum = 1 }; -static void * taosAsyncOutputLog(void *param); +static SLogObj tsLogObj = {.fileNum = 1}; +static void *taosAsyncOutputLog(void *param); static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen); static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(int32_t oldFd); @@ -139,8 +139,8 @@ static void taosStopLog() { void taosCloseLog() { taosStopLog(); - //tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); - taosMsleep(MAX_LOG_INTERVAL/1000); + // tsem_post(&(tsLogObj.logHandle->buffNotEmpty)); + taosMsleep(MAX_LOG_INTERVAL / 1000); if (taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { pthread_join(tsLogObj.logHandle->asyncThread, NULL); } @@ -217,7 +217,7 @@ static void *taosThreadToOpenNewFile(void *param) { tsLogObj.lines = 0; tsLogObj.openInProgress = 0; taosCloseLogByFd(oldFd); - + uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); taosPrintCfg(); @@ -308,9 +308,9 @@ static void taosGetLogFileName(char *fn) { static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { #ifdef WINDOWS /* - * always set maxFileNum to 1 - * means client log filename is unique in windows - */ + * always set maxFileNum to 1 + * means client log filename is unique in windows + */ maxFileNum = 1; #endif @@ -381,13 +381,14 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop print log.\n", tsLogDir, tsAvailLogDirGB, + tsTotalLogDirGB); fflush(stdout); return; } va_list argpointer; - char buffer[MAX_LOGLINE_BUFFER_SIZE] = { 0 }; + char buffer[MAX_LOGLINE_BUFFER_SIZE] = {0}; int32_t len; struct tm Tm, *ptm; struct timeval timeSecs; @@ -434,20 +435,20 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) { } } - if (dflag & DEBUG_SCREEN) - taosWriteFile(1, buffer, (uint32_t)len); + if (dflag & DEBUG_SCREEN) taosWriteFile(1, buffer, (uint32_t)len); if (dflag == 255) nInfo(buffer, len); } void taosDumpData(unsigned char *msg, int32_t len) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop dump log.\n", tsLogDir, tsAvailLogDirGB, + tsTotalLogDirGB); fflush(stdout); return; } - char temp[256]; - int32_t i, pos = 0, c = 0; + char temp[256]; + int32_t i, pos = 0, c = 0; for (i = 0; i < len; ++i) { sprintf(temp + pos, "%02x ", msg[i]); @@ -468,7 +469,8 @@ void taosDumpData(unsigned char *msg, int32_t len) { void taosPrintLongString(const char *flags, int32_t dflag, const char *format, ...) { if (tsTotalLogDirGB != 0 && tsAvailLogDirGB < tsMinimalLogDirGB) { - printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, tsTotalLogDirGB); + printf("server disk:%s space remain %.3f GB, total %.1f GB, stop write log.\n", tsLogDir, tsAvailLogDirGB, + tsTotalLogDirGB); fflush(stdout); return; } @@ -503,7 +505,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, . } else { taosWriteFile(tsLogObj.logHandle->fd, buffer, len); } - + if (tsLogObj.maxLines > 0) { atomic_add_fetch_32(&tsLogObj.lines, 1); @@ -542,7 +544,7 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { tLogBuff->stop = 0; if (pthread_mutex_init(&LOG_BUF_MUTEX(tLogBuff), NULL) < 0) goto _err; - //tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); + // tsem_init(&(tLogBuff->buffNotEmpty), 0, 0); return tLogBuff; @@ -576,12 +578,12 @@ static void taosCopyLogBuffer(SLogBuff *tLogBuff, int32_t start, int32_t end, ch } static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) { - int32_t start = 0; - int32_t end = 0; - int32_t remainSize = 0; + int32_t start = 0; + int32_t end = 0; + int32_t remainSize = 0; static int64_t lostLine = 0; - char tmpBuf[40] = {0}; - int32_t tmpBufLen = 0; + char tmpBuf[40] = {0}; + int32_t tmpBufLen = 0; if (tLogBuff == NULL || tLogBuff->stop) return -1; @@ -592,7 +594,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) remainSize = (start > end) ? (start - end - 1) : (start + LOG_BUF_SIZE(tLogBuff) - end - 1); if (lostLine > 0) { - sprintf(tmpBuf, "...Lost %"PRId64" lines here...\n", lostLine); + sprintf(tmpBuf, "...Lost %" PRId64 " lines here...\n", lostLine); tmpBufLen = (int32_t)strlen(tmpBuf); } @@ -610,7 +612,7 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) taosCopyLogBuffer(tLogBuff, LOG_BUF_START(tLogBuff), LOG_BUF_END(tLogBuff), msg, msgLen); - //int32_t w = atomic_sub_fetch_32(&waitLock, 1); + // int32_t w = atomic_sub_fetch_32(&waitLock, 1); /* if (w <= 0 || ((remainSize - msgLen - tmpBufLen) < (LOG_BUF_SIZE(tLogBuff) * 4 /5))) { tsem_post(&(tLogBuff->buffNotEmpty)); @@ -622,7 +624,6 @@ static int32_t taosPushLogBuffer(SLogBuff *tLogBuff, char *msg, int32_t msgLen) pthread_mutex_unlock(&LOG_BUF_MUTEX(tLogBuff)); - return 0; } @@ -634,9 +635,9 @@ static int32_t taosGetLogRemainSize(SLogBuff *tLogBuff, int32_t start, int32_t e static void taosWriteLog(SLogBuff *tLogBuff) { static int32_t lastDuration = 0; - int32_t remainChecked = 0; - int32_t start, end, pollSize; - + int32_t remainChecked = 0; + int32_t start, end, pollSize; + do { if (remainChecked == 0) { start = LOG_BUF_START(tLogBuff); @@ -662,24 +663,24 @@ static void taosWriteLog(SLogBuff *tLogBuff) { if (start < end) { taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, pollSize); } else { - int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); + int32_t tsize = LOG_BUF_SIZE(tLogBuff) - start; + taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff) + start, tsize); - taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); + taosWriteFile(tLogBuff->fd, LOG_BUF_BUFFER(tLogBuff), end); } dbgWN++; - dbgWSize+=pollSize; - + dbgWSize += pollSize; + if (pollSize < tLogBuff->minBuffSize) { dbgSmallWN++; if (writeInterval < MAX_LOG_INTERVAL) { writeInterval += LOG_INTERVAL_STEP; } - } else if (pollSize > LOG_BUF_SIZE(tLogBuff)/3) { + } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 3) { dbgBigWN++; writeInterval = MIN_LOG_INTERVAL; - } else if (pollSize > LOG_BUF_SIZE(tLogBuff)/4) { + } else if (pollSize > LOG_BUF_SIZE(tLogBuff) / 4) { if (writeInterval > MIN_LOG_INTERVAL) { writeInterval -= LOG_INTERVAL_STEP; } @@ -698,13 +699,13 @@ static void taosWriteLog(SLogBuff *tLogBuff) { writeInterval = MIN_LOG_INTERVAL; remainChecked = 1; - }while (1); + } while (1); } static void *taosAsyncOutputLog(void *param) { SLogBuff *tLogBuff = (SLogBuff *)param; setThreadName("log"); - + while (1) { taosMsleep(writeInterval); @@ -721,8 +722,8 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) { int32_t compressSize = 163840; int32_t ret = 0; int32_t len = 0; - char * data = malloc(compressSize); - FILE * srcFp = NULL; + char *data = malloc(compressSize); + FILE *srcFp = NULL; gzFile dstFp = NULL; srcFp = fopen(srcFileName, "r"); diff --git a/source/util/src/tthread.c b/source/util/src/tthread.c index 5ed7fb5aa00d1a5c5c3fffae2de46dbea57e011f..44fce1c882c4f0d7d66f36d378db1594e59f19ea 100644 --- a/source/util/src/tthread.c +++ b/source/util/src/tthread.c @@ -13,16 +13,16 @@ * along with this program. If not, see . */ -#include "os.h" #include "tthread.h" +#include "os.h" +#include "taoserror.h" #include "tdef.h" #include "tutil.h" #include "ulog.h" -#include "taoserror.h" // create new thread -pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param) { - pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); +pthread_t* taosCreateThread(void* (*__start_routine)(void*), void* param) { + pthread_t* pthread = (pthread_t*)malloc(sizeof(pthread_t)); pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -36,26 +36,24 @@ pthread_t* taosCreateThread( void *(*__start_routine) (void *), void* param) { return pthread; } -// destory thread +// destory thread bool taosDestoryThread(pthread_t* pthread) { - if(pthread == NULL) return false; - if(taosThreadRunning(pthread)) { + if (pthread == NULL) return false; + if (taosThreadRunning(pthread)) { pthread_cancel(*pthread); pthread_join(*pthread, NULL); } - + free(pthread); return true; } // thread running return true bool taosThreadRunning(pthread_t* pthread) { - if(pthread == NULL) return false; + if (pthread == NULL) return false; int ret = pthread_kill(*pthread, 0); - if(ret == ESRCH) - return false; - if(ret == EINVAL) - return false; + if (ret == ESRCH) return false; + if (ret == EINVAL) return false; // alive return true; } diff --git a/tests/script/general/user/monitor.sim b/tests/script/general/user/monitor.sim deleted file mode 100644 index 90aad59932115e04263da8cc1e8b91effb09eb07..0000000000000000000000000000000000000000 --- a/tests/script/general/user/monitor.sim +++ /dev/null @@ -1,32 +0,0 @@ -system sh/stop_dnodes.sh -system sh/deploy.sh -n dnode1 -i 1 - -print ========== step1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c monitor -v 1 -system sh/cfg.sh -n dnode1 -c monitorInterval -v 1 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ========== step2 -#sql drop database log -x step21 -# return -1 -#step21: -sql drop table log.dn -x step22 -# return -1 -step22: -sql drop user log -x step23 -# return -1 -step23: - -print ========== step3 - -sleep 2000 -#sql select * from log.dn -#if $rows == 0 then -# return -1 -#endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT