diff --git a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp b/source/dnode/mgmt/impl/test/qnode/dqnode.cpp index e609783eb02715db2c8b95d64054f76f26806a90..e64a0543fc358611cae3d233bf481728ec58a15a 100644 --- a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/impl/test/qnode/dqnode.cpp @@ -73,7 +73,7 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { } } -TEST_F(DndTestQnode, 01_Drop_Qnode) { +TEST_F(DndTestQnode, 02_Drop_Qnode) { { int32_t contLen = sizeof(SDDropQnodeReq); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index a001719dc44cf9696c2afbe3d5bf284438c04cee..a55e0dd2b252c328221d7b1e6d644eadf8e5db68 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -65,8 +65,8 @@ typedef enum { TRN_STAGE_PREPARE = 0, TRN_STAGE_REDO_LOG = 1, TRN_STAGE_REDO_ACTION = 2, - TRN_STAGE_COMMIT_LOG = 3, - TRN_STAGE_COMMIT = 4, + TRN_STAGE_COMMIT = 3, + TRN_STAGE_COMMIT_LOG = 4, TRN_STAGE_UNDO_ACTION = 5, TRN_STAGE_UNDO_LOG = 6, TRN_STAGE_ROLLBACK = 7, diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index bf1697fb19016803096f87aae367240f167f6eb2..47d0ce41055f58c965a94620edbeaa8c7ddb077b 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -54,7 +54,7 @@ static int32_t mndRestoreWal(SMnode *pMnode) { int64_t first = walGetFirstVer(pWal); int64_t last = walGetLastVer(pWal); - mDebug("restore sdb wal start, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); + mDebug("start to restore sdb wal, sdb ver:%" PRId64 ", wal first:%" PRId64 " last:%" PRId64, lastSdbVer, first, last); first = MAX(lastSdbVer + 1, first); for (int64_t ver = first; ver >= 0 && ver <= last; ++ver) { @@ -71,6 +71,7 @@ static int32_t mndRestoreWal(SMnode *pMnode) { goto WAL_RESTORE_OVER; } + mTrace("wal:%" PRId64 ", will be restored, content:%p", ver, pHead->head.body); if (sdbWriteNotFree(pSdb, (void *)pHead->head.body) < 0) { mError("failed to read wal from sdb since %s, ver:%" PRId64, terrstr(), ver); goto WAL_RESTORE_OVER; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index c1686c292398ee54ec54d0287702a214f27f6090..4a42133ce33d07523393be1023216b1153e6c7e3 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -24,7 +24,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); -static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOldTrans); +static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld); static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); @@ -112,6 +112,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { int32_t dataPos = 0; SDB_SET_INT32(pRaw, dataPos, pTrans->id, TRANS_ENCODE_OVER) SDB_SET_INT8(pRaw, dataPos, pTrans->policy, TRANS_ENCODE_OVER) + SDB_SET_INT8(pRaw, dataPos, pTrans->stage, TRANS_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, redoLogNum, TRANS_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, undoLogNum, TRANS_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, commitLogNum, TRANS_ENCODE_OVER) @@ -216,6 +217,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTrans->id, TRANS_DECODE_OVER) SDB_GET_INT8(pRaw, dataPos, (int8_t *)&pTrans->policy, TRANS_DECODE_OVER) + SDB_GET_INT8(pRaw, dataPos, (int8_t *)&pTrans->stage, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &redoLogNum, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &undoLogNum, TRANS_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &commitLogNum, TRANS_DECODE_OVER) @@ -314,9 +316,12 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { return 0; } -static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewTrans) { - mTrace("trans:%d, perform update action, old_row:%p new_row:%p", pOldTrans->id, pOldTrans, pNewTrans); - pOldTrans->stage = pNewTrans->stage; +static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOld, STrans *pNew) { + if (pNew->stage == TRN_STAGE_COMMIT) pNew->stage = TRN_STAGE_COMMIT_LOG; + + mTrace("trans:%d, perform update action, old row:%p stage:%d, new row:%p stage:%d", pOld->id, pOld, pOld->stage, pNew, + pNew->stage); + pOld->stage = pNew->stage; return 0; } @@ -464,16 +469,16 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { } mDebug("trans:%d, prepare finished", pTrans->id); - STrans *pNewTrans = mndAcquireTrans(pMnode, pTrans->id); - if (pNewTrans == NULL) { + STrans *pNew = mndAcquireTrans(pMnode, pTrans->id); + if (pNew == NULL) { mError("trans:%d, failed to read from sdb since %s", pTrans->id, terrstr()); return -1; } - pNewTrans->rpcHandle = pTrans->rpcHandle; - pNewTrans->rpcAHandle = pTrans->rpcAHandle; - mndTransExecute(pMnode, pNewTrans); - mndReleaseTrans(pMnode, pNewTrans); + pNew->rpcHandle = pTrans->rpcHandle; + pNew->rpcAHandle = pTrans->rpcAHandle; + mndTransExecute(pMnode, pNew); + mndReleaseTrans(pMnode, pNew); return 0; } @@ -645,7 +650,7 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr pAction->msgReceived = 0; pAction->errCode = 0; } else { - mDebug("trans:%d, action:%d not sent since %s", pTrans->id, action, terrstr()); + mDebug("trans:%d, action:%d not send since %s", pTrans->id, action, terrstr()); return -1; } } diff --git a/source/dnode/mnode/impl/test/bnode/bnode.cpp b/source/dnode/mnode/impl/test/bnode/bnode.cpp index e32c0e7f91c9e1f8470d5d8d1a346ed5587938c5..c33776fd1be3dbeb062592d6463365065e971c55 100644 --- a/source/dnode/mnode/impl/test/bnode/bnode.cpp +++ b/source/dnode/mnode/impl/test/bnode/bnode.cpp @@ -186,7 +186,6 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -199,7 +198,6 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -211,7 +209,7 @@ TEST_F(MndTestBnode, 03_Create_Bnode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateBnodeReq); @@ -250,7 +248,6 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { SMCreateBnodeReq* pReq = (SMCreateBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -263,7 +260,6 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { SMDropBnodeReq* pReq = (SMDropBnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -275,7 +271,7 @@ TEST_F(MndTestBnode, 04_Drop_Bnode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateBnodeReq); diff --git a/source/dnode/mnode/impl/test/qnode/qnode.cpp b/source/dnode/mnode/impl/test/qnode/qnode.cpp index 55806dca4ddbd30c3a4be3a62a4d5158f673e256..9aa41ca1771ce3c621c957da7a3d04db7aa7afe6 100644 --- a/source/dnode/mnode/impl/test/qnode/qnode.cpp +++ b/source/dnode/mnode/impl/test/qnode/qnode.cpp @@ -186,7 +186,6 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -199,7 +198,6 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -211,7 +209,7 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateQnodeReq); @@ -250,7 +248,6 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -263,7 +260,6 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { SMDropQnodeReq* pReq = (SMDropQnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -275,7 +271,7 @@ TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateQnodeReq); diff --git a/source/dnode/mnode/impl/test/snode/snode.cpp b/source/dnode/mnode/impl/test/snode/snode.cpp index 9e07be842f4c060721f8098bba5e805b26cc0109..2dcfc658a4c2ddf2bfceebb4ade8ef0132c16c9f 100644 --- a/source/dnode/mnode/impl/test/snode/snode.cpp +++ b/source/dnode/mnode/impl/test/snode/snode.cpp @@ -186,7 +186,6 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -199,7 +198,6 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_CREATING); @@ -211,7 +209,7 @@ TEST_F(MndTestSnode, 03_Create_Snode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateSnodeReq); @@ -250,7 +248,6 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMCreateSnodeReq* pReq = (SMCreateSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -263,7 +260,6 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { SMDropSnodeReq* pReq = (SMDropSnodeReq*)rpcMallocCont(contLen); pReq->dnodeId = htonl(2); - server2.Stop(); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_SDB_OBJ_DROPPING); @@ -275,7 +271,7 @@ TEST_F(MndTestSnode, 04_Drop_Snode_Rollback) { taosMsleep(1000); int32_t retry = 0; - int32_t retryMax = 10; + int32_t retryMax = 20; for (retry = 0; retry < retryMax; retry++) { int32_t contLen = sizeof(SMCreateSnodeReq); diff --git a/source/dnode/mnode/impl/test/trans/trans.cpp b/source/dnode/mnode/impl/test/trans/trans.cpp index 37c426f573d1510128e308a69a9ef0f852a4a5bc..e2ed707a99d540bb8e6bf168516cc9c77ed7dd7c 100644 --- a/source/dnode/mnode/impl/test/trans/trans.cpp +++ b/source/dnode/mnode/impl/test/trans/trans.cpp @@ -10,12 +10,21 @@ */ #include "sut.h" -#include "os.h" class MndTestTrans : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_trans", 9013); } - static void TearDownTestSuite() { test.Cleanup(); } + static void SetUpTestSuite() { + test.Init("/tmp/mnode_test_trans", 9013); + const char* fqdn = "localhost"; + const char* firstEp = "localhost:9013"; + server2.Start("/tmp/mnode_test_trans2", fqdn, 9020, firstEp); + } + + static void TearDownTestSuite() { + server2.Stop(); + test.Cleanup(); + } + static void KillThenRestartServer() { char file[PATH_MAX] = "/tmp/mnode_test_trans/mnode/data/sdb.data"; FileFd fd = taosOpenFileRead(file); @@ -41,14 +50,16 @@ class MndTestTrans : public ::testing::Test { test.ServerStart(); } - static Testbase test; + static Testbase test; + static TestServer server2; public: void SetUp() override {} void TearDown() override {} }; -Testbase MndTestTrans::test; +Testbase MndTestTrans::test; +TestServer MndTestTrans::server2; TEST_F(MndTestTrans, 01_Create_User_Crash) { { @@ -84,3 +95,97 @@ TEST_F(MndTestTrans, 01_Create_User_Crash) { CheckBinary("root", TSDB_USER_LEN); CheckBinary("root", TSDB_USER_LEN); } + +TEST_F(MndTestTrans, 02_Create_Qnode1_Crash) { + { + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + } + + KillThenRestartServer(); + { + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(1); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_QNODE_ALREADY_EXIST); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + } +} + +TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) { + { + int32_t contLen = sizeof(SCreateDnodeMsg); + + SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen); + strcpy(pReq->fqdn, "localhost"); + pReq->port = htonl(9020); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + taosMsleep(1300); + test.SendShowMetaReq(TSDB_MGMT_TABLE_DNODE, ""); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 2); + } + + { + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + server2.Stop(); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); + } + + KillThenRestartServer(); + + server2.DoStart(); + + { + int32_t retry = 0; + int32_t retryMax = 20; + + for (retry = 0; retry < retryMax; retry++) { + int32_t contLen = sizeof(SMCreateQnodeReq); + + SMCreateQnodeReq* pReq = (SMCreateQnodeReq*)rpcMallocCont(contLen); + pReq->dnodeId = htonl(2); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + if (pRsp->code == 0) break; + taosMsleep(1000); + } + + ASSERT_NE(retry, retryMax); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_QNODE, ""); + CHECK_META("show qnodes", 3); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 2); + } +} \ No newline at end of file