diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 09e93fda1cd4c4eb4ae764e2e451f3331d9df7cb..5a4b7555bf630bad4fc537eae48c3ddce0b8689e 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -34,11 +34,11 @@ extern "C" { typedef struct SSyncIO { STaosQueue *pMsgQ; - STaosQset *pQset; + STaosQset * pQset; pthread_t consumerTid; - void *serverRpc; - void *clientRpc; + void * serverRpc; + void * clientRpc; SEpSet myAddr; tmr_h qTimer; diff --git a/source/libs/sync/inc/syncRaftStore.h b/source/libs/sync/inc/syncRaftStore.h index 62bb94c8343ebf1c3bd45572c13131d73bc6f3f2..355a08ac8458fc69f9e5296086e66387dafe6026 100644 --- a/source/libs/sync/inc/syncRaftStore.h +++ b/source/libs/sync/inc/syncRaftStore.h @@ -43,13 +43,14 @@ int32_t raftStorePersist(SRaftStore *pRaftStore); int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len); int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len); -bool raftStoreHasVoted(SRaftStore *pRaftStore); -void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId); -void raftStoreClearVote(SRaftStore *pRaftStore); -void raftStoreNextTerm(SRaftStore *pRaftStore); -void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); -cJSON *raftStore2Json(SRaftStore *pRaftStore); -char *raftStore2Str(SRaftStore *pRaftStore); +bool raftStoreHasVoted(SRaftStore *pRaftStore); +void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId); +void raftStoreClearVote(SRaftStore *pRaftStore); +void raftStoreNextTerm(SRaftStore *pRaftStore); +void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term); +int32_t raftStoreFromJson(SRaftStore *pRaftStore, cJSON *pJson); +cJSON * raftStore2Json(SRaftStore *pRaftStore); +char * raftStore2Str(SRaftStore *pRaftStore); // for debug ------------------- void raftStorePrint(SRaftStore *pObj); diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 77c3d076988762713a2ffe354c1e91ca54b2e8bd..12e20ef7627323ee3d4c846cd3d660204360e26f 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -50,6 +50,7 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { } int32_t syncNodeElect(SSyncNode* pSyncNode) { + int32_t ret = 0; if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { syncNodeFollower2Candidate(pSyncNode); } @@ -62,7 +63,7 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { votesRespondReset(pSyncNode->pVotesRespond, pSyncNode->pRaftStore->currentTerm); syncNodeVoteForSelf(pSyncNode); - int32_t ret = syncNodeRequestVotePeers(pSyncNode); + ret = syncNodeRequestVotePeers(pSyncNode); assert(ret == 0); syncNodeResetElectTimer(pSyncNode); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 8176ac417a164ebcf06eb54b34a7219c07e64cb9..e6a43f2bb50bfc898b214ab8be77d4ce71e3a365 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -29,7 +29,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void *syncIOConsumerFunc(void *param); +static void * syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -234,9 +234,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO *io = param; + SSyncIO * io = param; STaosQall *qall; - SRpcMsg *pRpcMsg, rpcMsg; + SRpcMsg * pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 58a61ba77e0c882215d4a4f1e4ffada3e37c33ef..aa97104180c37dfb2455a79cfef8696717ac5ec1 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -96,7 +96,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 48a57dc0fb4e283cdac04083278e878cf8ed5e66..58710d7d1f88de8937aacb6b153e6a1fc9f785c8 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -103,6 +103,12 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { assert(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); + if (taosMkDir(pSyncInfo->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + sError("failed to create dir:%s since %s", pSyncInfo->path, terrstr()); + return NULL; + } + // init by SSyncInfo pSyncNode->vgId = pSyncInfo->vgId; pSyncNode->syncCfg = pSyncInfo->syncCfg; @@ -200,6 +206,9 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { pSyncNode->FpOnAppendEntriesReply = syncNodeOnAppendEntriesReplyCb; pSyncNode->FpOnTimeout = syncNodeOnTimeoutCb; + // start raft + syncNodeBecomeFollower(pSyncNode); + return pSyncNode; } diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 9a5401e3f1dc808a03ac60b4f545a963b1d26ab9..bb0eab0fdc559a05636a076d21fa963e5f34f2c7 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -97,16 +97,32 @@ int32_t raftStorePersist(SRaftStore *pRaftStore) { return 0; } -static bool raftStoreFileExist(char *path) { return taosStatFile(path, NULL, NULL) >= 0; } +static bool raftStoreFileExist(char *path) { + bool b = taosStatFile(path, NULL, NULL) >= 0; + return b; +} int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { assert(pRaftStore != NULL); cJSON *pRoot = cJSON_CreateObject(); - cJSON_AddNumberToObject(pRoot, "current_term", pRaftStore->currentTerm); - cJSON_AddNumberToObject(pRoot, "vote_for_addr", pRaftStore->voteFor.addr); + + char u64Buf[128]; + snprintf(u64Buf, sizeof(u64Buf), "%lu", pRaftStore->currentTerm); + cJSON_AddStringToObject(pRoot, "current_term", u64Buf); + + snprintf(u64Buf, sizeof(u64Buf), "%lu", pRaftStore->voteFor.addr); + cJSON_AddStringToObject(pRoot, "vote_for_addr", u64Buf); + cJSON_AddNumberToObject(pRoot, "vote_for_vgid", pRaftStore->voteFor.vgId); + uint64_t u64 = pRaftStore->voteFor.addr; + char host[128]; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pRoot, "addr_host", host); + cJSON_AddNumberToObject(pRoot, "addr_port", port); + char *serialized = cJSON_Print(pRoot); int len2 = strlen(serialized); assert(len2 < len); @@ -125,10 +141,12 @@ int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { cJSON *pRoot = cJSON_Parse(buf); cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); - pRaftStore->currentTerm = pCurrentTerm->valueint; + assert(cJSON_IsString(pCurrentTerm)); + sscanf(pCurrentTerm->valuestring, "%lu", &(pRaftStore->currentTerm)); cJSON *pVoteForAddr = cJSON_GetObjectItem(pRoot, "vote_for_addr"); - pRaftStore->voteFor.addr = pVoteForAddr->valueint; + assert(cJSON_IsString(pVoteForAddr)); + sscanf(pVoteForAddr->valuestring, "%lu", &(pRaftStore->voteFor.addr)); cJSON *pVoteForVgid = cJSON_GetObjectItem(pRoot, "vote_for_vgid"); pRaftStore->voteFor.vgId = pVoteForVgid->valueint; @@ -139,11 +157,10 @@ int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { bool raftStoreHasVoted(SRaftStore *pRaftStore) { bool b = syncUtilEmptyId(&(pRaftStore->voteFor)); - return b; + return (!b); } void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId) { - assert(!raftStoreHasVoted(pRaftStore)); assert(!syncUtilEmptyId(pRaftId)); pRaftStore->voteFor = *pRaftId; raftStorePersist(pRaftStore); @@ -164,6 +181,8 @@ void raftStoreSetTerm(SRaftStore *pRaftStore, SyncTerm term) { raftStorePersist(pRaftStore); } +int32_t raftStoreFromJson(SRaftStore *pRaftStore, cJSON *pJson) { return 0; } + cJSON *raftStore2Json(SRaftStore *pRaftStore) { char u64buf[128]; cJSON *pRoot = cJSON_CreateObject(); @@ -185,6 +204,9 @@ cJSON *raftStore2Json(SRaftStore *pRaftStore) { } cJSON_AddNumberToObject(pVoteFor, "vgId", pRaftStore->voteFor.vgId); cJSON_AddItemToObject(pRoot, "voteFor", pVoteFor); + + int hasVoted = raftStoreHasVoted(pRaftStore); + cJSON_AddNumberToObject(pRoot, "hasVoted", hasVoted); } cJSON *pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 5830a0a54f93f0b8100c41530cb8dea22329b806..00851a7b4fac76256e3659fb0b6a0980ca9e8d84 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -116,7 +116,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { char *voteGranted2Str(SVotesGranted *pVotesGranted) { cJSON *pJson = voteGranted2Json(pVotesGranted); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -238,7 +238,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { char *votesRespond2Str(SVotesRespond *pVotesRespond) { cJSON *pJson = votesRespond2Json(pVotesRespond); - char *serialized = cJSON_Print(pJson); + char * serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index b542f55ec3617a1f16b30a9f916c7c3d7fbcb2bc..2583deb32f6c4b3e25a426d29e6f46c821731b34 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -27,6 +27,7 @@ add_executable(syncPingReplyTest "") add_executable(syncRpcMsgTest "") add_executable(syncPingTimerTest2 "") add_executable(syncPingSelfTest "") +add_executable(syncElectTest "") target_sources(syncTest @@ -145,6 +146,10 @@ target_sources(syncPingSelfTest PRIVATE "syncPingSelfTest.cpp" ) +target_sources(syncElectTest + PRIVATE + "syncElectTest.cpp" +) target_include_directories(syncTest @@ -292,6 +297,16 @@ target_include_directories(syncPingSelfTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncElectTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncElectTest + PUBLIC + "${CMAKE_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -410,6 +425,10 @@ target_link_libraries(syncPingSelfTest sync gtest_main ) +target_link_libraries(syncElectTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e52fe61ef5e68719326400d29311f8c60b12603a --- /dev/null +++ b/source/libs/sync/test/syncElectTest.cpp @@ -0,0 +1,121 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 1; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; +SSyncNode* gSyncNode; + +SSyncNode* syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "%s", "./elect_test"); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + pWal = walOpen("./elect_test_wal", &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + SSyncNode* pSyncNode = syncNodeOpen(&syncInfo); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + return pSyncNode; +} + +SSyncNode* syncInitTest() { return syncNodeInit(); } + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncEnvStart(); + assert(ret == 0); + + gSyncNode = syncInitTest(); + assert(gSyncNode != NULL); + syncNodePrint2((char*)"", gSyncNode); + + initRaftId(gSyncNode); + + //--------------------------- + while (1) { + sTrace("while 1 sleep, state: %d, %s", gSyncNode->state, syncUtilState2String(gSyncNode->state)); + taosMsleep(1000); + } + + return 0; +} diff --git a/source/libs/sync/test/syncRaftStoreTest.cpp b/source/libs/sync/test/syncRaftStoreTest.cpp index 0c1c9b881ed0ecc7978d607d3e679f7f3f35ebc9..688802625ac42203b7f52a459b7440ff3324f8e8 100644 --- a/source/libs/sync/test/syncRaftStoreTest.cpp +++ b/source/libs/sync/test/syncRaftStoreTest.cpp @@ -3,6 +3,7 @@ #include #include "syncIO.h" #include "syncInt.h" +#include "syncUtil.h" void logTest() { sTrace("--- sync log test: trace"); @@ -13,6 +14,21 @@ void logTest() { sFatal("--- sync log test: fatal"); } +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 5; +int32_t myIndex = 0; +SRaftId ids[TSDB_MAX_REPLICA]; + +void initRaftId() { + for (int i = 0; i < replicaNum; ++i) { + ids[i].addr = syncUtilAddr2U64("127.0.0.1", ports[i]); + ids[i].vgId = 1234; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + int main() { // taosInitLog((char *)"syncTest.log", 100000, 10); tsAsyncLog = 0; @@ -20,23 +36,35 @@ int main() { logTest(); - SRaftStore *pRaftStore = raftStoreOpen("./raft_store.json"); + initRaftId(); + + SRaftStore* pRaftStore = raftStoreOpen("./test_raft_store.json"); assert(pRaftStore != NULL); - raftStorePrint(pRaftStore); - -#if 0 - pRaftStore->currentTerm = 100; - pRaftStore->voteFor.addr = 200; - pRaftStore->voteFor.vgId = 300; - raftStorePersist(pRaftStore); - raftStorePrint(pRaftStore); -#endif - - ++(pRaftStore->currentTerm); - ++(pRaftStore->voteFor.addr); - ++(pRaftStore->voteFor.vgId); - raftStorePersist(pRaftStore); - raftStorePrint(pRaftStore); + raftStorePrint2((char*)"==raftStoreOpen==", pRaftStore); + + raftStoreSetTerm(pRaftStore, 100); + raftStorePrint2((char*)"==raftStoreSetTerm==", pRaftStore); + + raftStoreVote(pRaftStore, &ids[0]); + raftStorePrint2((char*)"==raftStoreVote==", pRaftStore); + + raftStoreClearVote(pRaftStore); + raftStorePrint2((char*)"==raftStoreClearVote==", pRaftStore); + + raftStoreVote(pRaftStore, &ids[1]); + raftStorePrint2((char*)"==raftStoreVote==", pRaftStore); + + raftStoreNextTerm(pRaftStore); + raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); + + raftStoreNextTerm(pRaftStore); + raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); + + raftStoreNextTerm(pRaftStore); + raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); + + raftStoreNextTerm(pRaftStore); + raftStorePrint2((char*)"==raftStoreNextTerm==", pRaftStore); return 0; }