Merge pull request #11768 from taosdata/feature/sync-vnode-merge-3.0

Feature/sync vnode merge 3.0

include/common/tmsgdef.h

@@ -205,6 +205,21 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL)
 
+  // sync integration
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING_REPLY, "vnode-sync-ping-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST, "vnode-sync-client-request", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, "vnode-sync-client-request-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE, "vnode-sync-request-vote", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_REQUEST_VOTE_REPLY, "vnode-sync-request-vote-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES, "vnode-sync-append-entries", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, "vnode-sync-append-entries-reply", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_NOOP, "vnode-sync-noop", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL)
+  
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_COMPACT_VNODE, "vnode-compact-vnode", NULL, NULL)

include/libs/sync/sync.h

@@ -20,11 +20,14 @@
 extern "C" {
 #endif
 
+#include <stdbool.h>
 #include <stdint.h>
-#include <tdatablock.h>
-#include "taosdef.h"
-#include "trpc.h"
-#include "wal.h"
+//#include <tdatablock.h>
+#include "cJSON.h"
+#include "tdef.h"
+//#include "taosdef.h"
+//#include "trpc.h"
+//#include "wal.h"
 
 typedef uint64_t SyncNodeId;
 typedef int32_t  SyncGroupId;
@@ -38,14 +41,9 @@ typedef enum {
   TAOS_SYNC_STATE_ERROR = 103,
 } ESyncState;
 
-typedef struct SSyncBuffer {
-  void*  data;
-  size_t len;
-} SSyncBuffer;
-
 typedef struct SNodeInfo {
-  uint16_t nodePort;                 // node sync Port
-  char     nodeFqdn[TSDB_FQDN_LEN];  // node FQDN
+  uint16_t nodePort;
+  char     nodeFqdn[TSDB_FQDN_LEN];
 } SNodeInfo;
 
 typedef struct SSyncCfg {
@@ -54,47 +52,46 @@ typedef struct SSyncCfg {
   SNodeInfo nodeInfo[TSDB_MAX_REPLICA];
 } SSyncCfg;
 
-typedef struct SNodesRole {
-  int32_t    replicaNum;
-  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
-  ESyncState role[TSDB_MAX_REPLICA];
-} SNodesRole;
-
-// abstract definition of snapshot
 typedef struct SSnapshot {
   void*     data;
   SyncIndex lastApplyIndex;
+  SyncTerm  lastApplyTerm;
 } SSnapshot;
 
-typedef struct SSyncFSM {
-  void* data;
+typedef enum {
+  TAOS_SYNC_FSM_CB_SUCCESS = 0,
+  TAOS_SYNC_FSM_CB_OTHER_ERROR,
+} ESyncFsmCbCode;
 
-  // when value in pMsg finish a raft flow, FpCommitCb is called, code indicates the result
-  // user can do something according to the code and isWeak. for example, write data into tsdb
-  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                     ESyncState state);
+typedef struct SFsmCbMeta {
+  SyncIndex  index;
+  bool       isWeak;
+  int32_t    code;
+  ESyncState state;
+  uint64_t   seqNum;
+} SFsmCbMeta;
 
-  // when value in pMsg has been written into local log store, FpPreCommitCb is called, code indicates the result
-  // user can do something according to the code and isWeak. for example, write data into tsdb
-  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                        ESyncState state);
+struct SRpcMsg;
+typedef struct SRpcMsg SRpcMsg;
 
-  // when log entry is updated by a new one, FpRollBackCb is called
-  // user can do something to roll back. for example, delete data from tsdb, or just ignore it
-  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SyncIndex index, bool isWeak, int32_t code,
-                       ESyncState state);
+typedef struct SSyncFSM {
+  void* data;
 
-  // user should implement this function, use "data" to take snapshot into "snapshot"
-  int32_t (*FpTakeSnapshot)(SSnapshot* snapshot);
+  void (*FpCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
+  void (*FpPreCommitCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
+  void (*FpRollBackCb)(struct SSyncFSM* pFsm, const SRpcMsg* pMsg, SFsmCbMeta cbMeta);
 
-  // user should implement this function, restore "data" from "snapshot"
-  int32_t (*FpRestoreSnapshot)(const SSnapshot* snapshot);
+  int32_t (*FpGetSnapshot)(struct SSyncFSM* pFsm, SSnapshot* pSnapshot);
+  int32_t (*FpRestoreSnapshot)(struct SSyncFSM* pFsm, const SSnapshot* snapshot);
 
 } SSyncFSM;
 
 struct SSyncRaftEntry;
 typedef struct SSyncRaftEntry SSyncRaftEntry;
 
+#define SYNC_INDEX_BEGIN   0
+#define SYNC_INDEX_INVALID -1
+
 // abstract definition of log store in raft
 // SWal implements it
 typedef struct SSyncLogStore {
@@ -123,20 +120,11 @@ typedef struct SSyncLogStore {
 
 } SSyncLogStore;
 
-// raft need to persist two variables in storage: currentTerm, voteFor
-typedef struct SStateMgr {
-  void* data;
-
-  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
-  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+struct SWal;
+typedef struct SWal SWal;
 
-  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
-  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
-
-  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
-
-} SStateMgr;
+struct SEpSet;
+typedef struct SEpSet SEpSet;
 
 typedef struct SSyncInfo {
   SyncGroupId vgId;
@@ -152,27 +140,35 @@ typedef struct SSyncInfo {
 
 } SSyncInfo;
 
-struct SSyncNode;
-typedef struct SSyncNode SSyncNode;
+int32_t     syncInit();
+void        syncCleanUp();
+int64_t     syncOpen(const SSyncInfo* pSyncInfo);
+void        syncStart(int64_t rid);
+void        syncStop(int64_t rid);
+int32_t     syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
+ESyncState  syncGetMyRole(int64_t rid);
+const char* syncGetMyRoleStr(int64_t rid);
+SyncTerm    syncGetMyTerm(int64_t rid);
 
-int32_t syncInit();
-void    syncCleanUp();
+typedef enum {
+  TAOS_SYNC_PROPOSE_SUCCESS = 0,
+  TAOS_SYNC_PROPOSE_NOT_LEADER,
+  TAOS_SYNC_PROPOSE_OTHER_ERROR,
+} ESyncProposeCode;
 
-int64_t    syncStart(const SSyncInfo* pSyncInfo);
-void       syncStop(int64_t rid);
-int32_t    syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
-int32_t    syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
-ESyncState syncGetMyRole(int64_t rid);
+int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
 
-// propose with sequence number, to implement linearizable semantics
-int32_t syncPropose2(int64_t rid, const SRpcMsg* pMsg, bool isWeak, uint64_t seqNum);
+extern int32_t sDebugFlag;
 
-// for compatibility, the same as syncPropose
-int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
+//-----------------------------------------
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
 
-extern int32_t sDebugFlag;
+struct SSyncBuffer;
+typedef struct SSyncBuffer SSyncBuffer;
+//-----------------------------------------
 
-const char *syncStr(ESyncState state);
+const char* syncStr(ESyncState state);
 
 #ifdef __cplusplus
 }

include/libs/sync/syncTools.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_TOOLS_H
+#define _TD_LIBS_SYNC_TOOLS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+//#include <tdatablock.h>
+#include "cJSON.h"
+//#include "taosdef.h"
+#include "trpc.h"
+//#include "wal.h"
+
+// ------------------ ds -------------------
+typedef struct SRaftId {
+  SyncNodeId  addr;
+  SyncGroupId vgId;
+} SRaftId;
+
+// ------------------ control -------------------
+struct SSyncNode;
+typedef struct SSyncNode SSyncNode;
+
+SSyncNode* syncNodeAcquire(int64_t rid);
+void       syncNodeRelease(SSyncNode* pNode);
+
+int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg);
+void    syncSetQ(int64_t rid, void* queueHandle);
+void    syncSetRpc(int64_t rid, void* rpcHandle);
+char*   sync2SimpleStr(int64_t rid);
+
+// set timer ms
+void setPingTimerMS(int64_t rid, int32_t pingTimerMS);
+void setElectTimerMS(int64_t rid, int32_t electTimerMS);
+void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS);
+
+// for compatibility, the same as syncPropose
+int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak);
+
+// utils
+const char* syncUtilState2String(ESyncState state);
+
+// ------------------ for debug -------------------
+void syncRpcMsgPrint(SRpcMsg* pMsg);
+void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg);
+void syncRpcMsgLog(SRpcMsg* pMsg);
+void syncRpcMsgLog2(char* s, SRpcMsg* pMsg);
+
+// ------------------ for compile -------------------
+typedef struct SSyncBuffer {
+  void*  data;
+  size_t len;
+} SSyncBuffer;
+
+typedef struct SNodesRole {
+  int32_t    replicaNum;
+  SNodeInfo  nodeInfo[TSDB_MAX_REPLICA];
+  ESyncState role[TSDB_MAX_REPLICA];
+} SNodesRole;
+
+typedef struct SStateMgr {
+  void* data;
+
+  int32_t (*getCurrentTerm)(struct SStateMgr* pMgr, SyncTerm* pCurrentTerm);
+  int32_t (*persistCurrentTerm)(struct SStateMgr* pMgr, SyncTerm pCurrentTerm);
+
+  int32_t (*getVoteFor)(struct SStateMgr* pMgr, SyncNodeId* pVoteFor);
+  int32_t (*persistVoteFor)(struct SStateMgr* pMgr, SyncNodeId voteFor);
+
+  int32_t (*getSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+  int32_t (*persistSyncCfg)(struct SStateMgr* pMgr, SSyncCfg* pSyncCfg);
+
+} SStateMgr;
+
+// ------------------ for message process -------------------
+
+// ---------------------------------------------
+typedef struct SyncPing {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPing;
+
+SyncPing* syncPingBuild(uint32_t dataLen);
+SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void      syncPingDestroy(SyncPing* pMsg);
+void      syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
+void      syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
+char*     syncPingSerialize2(const SyncPing* pMsg, uint32_t* len);
+SyncPing* syncPingDeserialize2(const char* buf, uint32_t len);
+int32_t   syncPingSerialize3(const SyncPing* pMsg, char* buf, int32_t bufLen);
+SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen);
+void      syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
+void      syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
+SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*    syncPing2Json(const SyncPing* pMsg);
+char*     syncPing2Str(const SyncPing* pMsg);
+
+// for debug ----------------------
+void syncPingPrint(const SyncPing* pMsg);
+void syncPingPrint2(char* s, const SyncPing* pMsg);
+void syncPingLog(const SyncPing* pMsg);
+void syncPingLog2(char* s, const SyncPing* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncPingReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  uint32_t dataLen;
+  char     data[];
+} SyncPingReply;
+
+SyncPingReply* syncPingReplyBuild(uint32_t dataLen);
+SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, int32_t vgId, const char* str);
+SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId, int32_t vgId);
+void           syncPingReplyDestroy(SyncPingReply* pMsg);
+void           syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
+void           syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
+char*          syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len);
+SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len);
+int32_t        syncPingReplySerialize3(const SyncPingReply* pMsg, char* buf, int32_t bufLen);
+SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen);
+void           syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
+void           syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
+SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*         syncPingReply2Json(const SyncPingReply* pMsg);
+char*          syncPingReply2Str(const SyncPingReply* pMsg);
+
+// for debug ----------------------
+void syncPingReplyPrint(const SyncPingReply* pMsg);
+void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg);
+void syncPingReplyLog(const SyncPingReply* pMsg);
+void syncPingReplyLog2(char* s, const SyncPingReply* pMsg);
+
+// ---------------------------------------------
+typedef enum ESyncTimeoutType {
+  SYNC_TIMEOUT_PING = 100,
+  SYNC_TIMEOUT_ELECTION,
+  SYNC_TIMEOUT_HEARTBEAT,
+} ESyncTimeoutType;
+
+typedef struct SyncTimeout {
+  uint32_t         bytes;
+  int32_t          vgId;
+  uint32_t         msgType;
+  ESyncTimeoutType timeoutType;
+  uint64_t         logicClock;
+  int32_t          timerMS;
+  void*            data;  // need optimized
+} SyncTimeout;
+
+SyncTimeout* syncTimeoutBuild();
+SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, int32_t vgId,
+                               void* data);
+void         syncTimeoutDestroy(SyncTimeout* pMsg);
+void         syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen);
+void         syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg);
+char*        syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len);
+SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len);
+void         syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg);
+void         syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg);
+SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*       syncTimeout2Json(const SyncTimeout* pMsg);
+char*        syncTimeout2Str(const SyncTimeout* pMsg);
+
+// for debug ----------------------
+void syncTimeoutPrint(const SyncTimeout* pMsg);
+void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg);
+void syncTimeoutLog(const SyncTimeout* pMsg);
+void syncTimeoutLog2(char* s, const SyncTimeout* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequest {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;          // SyncClientRequest msgType
+  uint32_t originalRpcType;  // user RpcMsg msgType
+  uint64_t seqNum;
+  bool     isWeak;
+  uint32_t dataLen;  // user RpcMsg.contLen
+  char     data[];   // user RpcMsg.pCont
+} SyncClientRequest;
+
+SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
+SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak,
+                                           int32_t vgId);  // step 1
+void               syncClientRequestDestroy(SyncClientRequest* pMsg);
+void               syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
+void               syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
+char*              syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len);
+SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len);
+void               syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg);  // step 2
+void               syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
+SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg);  // step 3
+cJSON*             syncClientRequest2Json(const SyncClientRequest* pMsg);
+char*              syncClientRequest2Str(const SyncClientRequest* pMsg);
+
+// for debug ----------------------
+void syncClientRequestPrint(const SyncClientRequest* pMsg);
+void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg);
+void syncClientRequestLog(const SyncClientRequest* pMsg);
+void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncClientRequestReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  int32_t  errCode;
+  SRaftId  leaderHint;
+} SyncClientRequestReply;
+
+// ---------------------------------------------
+typedef struct SyncRequestVote {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex lastLogIndex;
+  SyncTerm  lastLogTerm;
+} SyncRequestVote;
+
+SyncRequestVote* syncRequestVoteBuild(int32_t vgId);
+void             syncRequestVoteDestroy(SyncRequestVote* pMsg);
+void             syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen);
+void             syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg);
+char*            syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len);
+SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len);
+void             syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg);
+void             syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg);
+SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*           syncRequestVote2Json(const SyncRequestVote* pMsg);
+char*            syncRequestVote2Str(const SyncRequestVote* pMsg);
+
+// for debug ----------------------
+void syncRequestVotePrint(const SyncRequestVote* pMsg);
+void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg);
+void syncRequestVoteLog(const SyncRequestVote* pMsg);
+void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncRequestVoteReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm term;
+  bool     voteGranted;
+} SyncRequestVoteReply;
+
+SyncRequestVoteReply* syncRequestVoteReplyBuild(int32_t vgId);
+void                  syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg);
+void                  syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen);
+void                  syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len);
+SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len);
+void                  syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg);
+void                  syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg);
+SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg);
+char*                 syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg);
+
+// for debug ----------------------
+void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg);
+void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntries {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  SyncIndex prevLogIndex;
+  SyncTerm  prevLogTerm;
+  SyncIndex commitIndex;
+  uint32_t  dataLen;
+  char      data[];
+} SyncAppendEntries;
+
+SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen, int32_t vgId);
+void               syncAppendEntriesDestroy(SyncAppendEntries* pMsg);
+void               syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen);
+void               syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg);
+char*              syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len);
+SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len);
+void               syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg);
+void               syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg);
+SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*             syncAppendEntries2Json(const SyncAppendEntries* pMsg);
+char*              syncAppendEntries2Str(const SyncAppendEntries* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesPrint(const SyncAppendEntries* pMsg);
+void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
+void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncAppendEntriesReply {
+  uint32_t bytes;
+  int32_t  vgId;
+  uint32_t msgType;
+  SRaftId  srcId;
+  SRaftId  destId;
+  // private data
+  SyncTerm  term;
+  bool      success;
+  SyncIndex matchIndex;
+} SyncAppendEntriesReply;
+
+SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId);
+void                    syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg);
+void                    syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen);
+void                    syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len);
+SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len);
+void                    syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg);
+void                    syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg);
+SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
+cJSON*                  syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg);
+char*                   syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg);
+
+// for debug ----------------------
+void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
+void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+typedef struct SyncApplyMsg {
+  uint32_t   bytes;
+  int32_t    vgId;
+  uint32_t   msgType;          // user SyncApplyMsg msgType
+  uint32_t   originalRpcType;  // user RpcMsg msgType
+  SFsmCbMeta fsmMeta;
+  uint32_t   dataLen;  // user RpcMsg.contLen
+  char       data[];   // user RpcMsg.pCont
+} SyncApplyMsg;
+
+SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen);
+SyncApplyMsg* syncApplyMsgBuild2(const SRpcMsg* pOriginalRpcMsg, int32_t vgId, SFsmCbMeta* pMeta);
+void          syncApplyMsgDestroy(SyncApplyMsg* pMsg);
+void          syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen);
+void          syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg);
+char*         syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len);
+SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len);
+void syncApplyMsg2RpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pRpcMsg);     // SyncApplyMsg to SRpcMsg, put it into ApplyQ
+void syncApplyMsgFromRpcMsg(const SRpcMsg* pRpcMsg, SyncApplyMsg* pMsg);  // get SRpcMsg from ApplyQ, to SyncApplyMsg
+SyncApplyMsg* syncApplyMsgFromRpcMsg2(const SRpcMsg* pRpcMsg);
+void syncApplyMsg2OriginalRpcMsg(const SyncApplyMsg* pMsg, SRpcMsg* pOriginalRpcMsg);  // SyncApplyMsg to OriginalRpcMsg
+cJSON* syncApplyMsg2Json(const SyncApplyMsg* pMsg);
+char*  syncApplyMsg2Str(const SyncApplyMsg* pMsg);
+
+// for debug ----------------------
+void syncApplyMsgPrint(const SyncApplyMsg* pMsg);
+void syncApplyMsgPrint2(char* s, const SyncApplyMsg* pMsg);
+void syncApplyMsgLog(const SyncApplyMsg* pMsg);
+void syncApplyMsgLog2(char* s, const SyncApplyMsg* pMsg);
+
+// on message ----------------------
+int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg);
+int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg);
+int32_t syncNodeOnTimeoutCb(SSyncNode* ths, SyncTimeout* pMsg);
+int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg);
+int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg);
+int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg);
+int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg);
+int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
+
+// ---------------------------------------------
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_TOOLS_H*/

include/libs/transport/trpc.h

@@ -38,11 +38,11 @@ typedef struct SRpcConnInfo {
 
 typedef struct SRpcMsg {
   tmsg_t  msgType;
-  void *  pCont;
+  void   *pCont;
   int     contLen;
   int32_t code;
-  void *  handle;         // rpc handle returned to app
-  void *  ahandle;        // app handle set by client
+  void   *handle;         // rpc handle returned to app
+  void   *ahandle;        // app handle set by client
   int     noResp;         // has response or not(default 0, 0: resp, 1: no resp);
   int     persistHandle;  // persist handle or not
 
@@ -54,8 +54,8 @@ typedef struct {
   uint16_t clientPort;
   SRpcMsg  rpcMsg;
   int32_t  rspLen;
-  void *   pRsp;
-  void *   pNode;
+  void    *pRsp;
+  void    *pNode;
 } SNodeMsg;
 
 typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *);
@@ -64,7 +64,7 @@ typedef int (*RpcRfp)(void *parent, SRpcMsg *, SEpSet *);
 
 typedef struct SRpcInit {
   uint16_t localPort;     // local port
-  char *   label;         // for debug purpose
+  char    *label;         // for debug purpose
   int      numOfThreads;  // number of threads to handle connections
   int      sessions;      // number of sessions allowed
   int8_t   connType;      // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
@@ -97,23 +97,23 @@ typedef struct {
 
 typedef struct {
   int32_t msgType;
-  void *  val;
+  void   *val;
   int32_t (*clone)(void *src, void **dst);
   void (*freeFunc)(const void *arg);
 } SRpcBrokenlinkVal;
 
 typedef struct {
-  SHashObj *        args;
+  SHashObj         *args;
   SRpcBrokenlinkVal brokenVal;
 } SRpcCtx;
 
 int32_t rpcInit();
 void    rpcCleanup();
-void *  rpcOpen(const SRpcInit *pRpc);
+void   *rpcOpen(const SRpcInit *pRpc);
 void    rpcClose(void *);
-void *  rpcMallocCont(int contLen);
+void   *rpcMallocCont(int contLen);
 void    rpcFreeCont(void *pCont);
-void *  rpcReallocCont(void *ptr, int contLen);
+void   *rpcReallocCont(void *ptr, int contLen);
 
 // Because taosd supports multi-process mode
 // These functions should not be used on the server side

include/util/taoserror.h

@@ -411,6 +411,10 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_SYN_INVALID_MSGLEN            TAOS_DEF_ERROR_CODE(0, 0x0909)
 #define TSDB_CODE_SYN_INVALID_MSGTYPE           TAOS_DEF_ERROR_CODE(0, 0x090A)
 
+// sync integration
+#define TSDB_CODE_SYN_NOT_LEADER                TAOS_DEF_ERROR_CODE(0, 0x0910)
+#define TSDB_CODE_SYN_INTERNAL_ERROR            TAOS_DEF_ERROR_CODE(0, 0x09FF)
+
 // tq
 #define TSDB_CODE_TQ_INVALID_CONFIG             TAOS_DEF_ERROR_CODE(0, 0x0A00)
 #define TSDB_CODE_TQ_INIT_FAILED                TAOS_DEF_ERROR_CODE(0, 0x0A01)

source/dnode/mgmt/mgmt_vnode/inc/vmInt.h

@@ -105,6 +105,7 @@ void    vmStopWorker(SVnodesMgmt *pMgmt);
 int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
 void    vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode);
 
+int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc);  // sync integration
 int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
 int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);

source/dnode/mgmt/mgmt_vnode/src/vmHandle.c

@@ -121,6 +121,16 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->hashBegin = pCreate->hashBegin;
   pCfg->hashEnd = pCreate->hashEnd;
   pCfg->hashMethod = pCreate->hashMethod;
+
+  // sync integration
+  pCfg->syncCfg.myIndex = pCreate->selfIndex;
+  pCfg->syncCfg.replicaNum = pCreate->replica;
+  memset(&(pCfg->syncCfg.nodeInfo), 0, sizeof(pCfg->syncCfg.nodeInfo));
+  for (int i = 0; i < pCreate->replica; ++i) {
+    (pCfg->syncCfg.nodeInfo)[i].nodePort = (pCreate->replicas)[i].port;
+    snprintf((pCfg->syncCfg.nodeInfo)[i].nodeFqdn, sizeof((pCfg->syncCfg.nodeInfo)[i].nodeFqdn), "%s",
+             (pCreate->replicas)[i].fqdn);
+  }
 }
 
 static void vmGenerateWrapperCfg(SVnodesMgmt *pMgmt, SCreateVnodeReq *pCreate, SWrapperCfg *pCfg) {
@@ -172,6 +182,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) {
   msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
   msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
   msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
+  msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue;  // sync integration
   msgCb.qsizeFp = vmGetQueueSize;
 
   SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
@@ -252,7 +263,7 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, vmProcessQueryMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_RES_READY, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, vmProcessFetchMsg, DEFAULT_HANDLE);
@@ -265,11 +276,11 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
   dmSetMsgHandle(pWrapper, TDMT_VND_CREATE_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CANCEL_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_DROP_SMA, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
-  //dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE);
-  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);  
+  // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, vmProcessWriteMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_CANCEL_CONN, vmProcessWriteMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, vmProcessFetchMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_MQ_VG_CHANGE, (NodeMsgFp)vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_CONSUME, vmProcessFetchMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, vmProcessWriteMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, vmProcessFetchMsg, DEFAULT_HANDLE);
@@ -283,4 +294,17 @@ void vmInitMsgHandle(SMgmtWrapper *pWrapper) {
 
   dmSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
   dmSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+  // dmSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, vmProcessMgmtMsg, DEFAULT_HANDLE);
+
+  // sync integration
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_TIMEOUT, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_PING_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_CLIENT_REQUEST_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_REQUEST_VOTE_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
+  dmSetMsgHandle(pWrapper, TDMT_VND_SYNC_APPEND_ENTRIES_REPLY, (NodeMsgFp)vmProcessSyncMsg, DEFAULT_HANDLE);
 }

source/dnode/mgmt/mgmt_vnode/src/vmInt.c

@@ -73,6 +73,12 @@ int32_t vmOpenVnode(SVnodesMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) {
     return -1;
   }
 
+  // sync integration
+  vnodeSyncSetQ(pImpl, NULL);
+  vnodeSyncSetRpc(pImpl, NULL);
+  int32_t ret = vnodeSyncStart(pImpl);
+  assert(ret == 0);
+
   taosWLockLatch(&pMgmt->latch);
   int32_t code = taosHashPut(pMgmt->hash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *));
   taosWUnLockLatch(&pMgmt->latch);
@@ -137,6 +143,7 @@ static void *vmOpenVnodeFunc(void *param) {
     msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue;
     msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue;
     msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue;
+    msgCb.queueFps[SYNC_QUEUE] = vmPutMsgToSyncQueue;  // sync integration
     msgCb.qsizeFp = vmGetQueueSize;
     snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, pCfg->vgId);
     SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, msgCb);
@@ -266,6 +273,9 @@ static void vmCleanup(SMgmtWrapper *pWrapper) {
   // walCleanUp();
   taosMemoryFree(pMgmt);
   pWrapper->pMgmt = NULL;
+
+  syncCleanUp();
+
   dInfo("vnode-mgmt is cleaned up");
 }
 
@@ -306,6 +316,12 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
   }
   dmReportStartup(pDnode, "vnode-wal", "initialized");
 
+  // sync integration
+  if (syncInit() != 0) {
+    dError("failed to open sync since %s", terrstr());
+    return -1;
+  }
+
   if (vnodeInit(tsNumOfCommitThreads) != 0) {
     dError("failed to init vnode since %s", terrstr());
     goto _OVER;

source/dnode/mgmt/mgmt_vnode/src/vmWorker.c

@@ -14,8 +14,12 @@
  */
 
 #define _DEFAULT_SOURCE
+
 #include "vmInt.h"
 
+#include "sync.h"
+#include "syncTools.h"
+
 static inline void vmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) {
   SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle,
                  .ahandle = pMsg->rpcMsg.ahandle,
@@ -88,7 +92,7 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, SNodeMsg *pMsg) {
 
 static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
-  int64_t    version;
+  SRpcMsg    rsp;
 
   SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SNodeMsg *));
   if (pArray == NULL) {
@@ -107,13 +111,15 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
     }
   }
 
+#if 0
+  int64_t version;
+
   vnodePreprocessWriteReqs(pVnode->pImpl, pArray, &version);
 
   numOfMsgs = taosArrayGetSize(pArray);
   for (int32_t i = 0; i < numOfMsgs; i++) {
     SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
     SRpcMsg  *pRpc = &pMsg->rpcMsg;
-    SRpcMsg   rsp;
 
     rsp.pCont = NULL;
     rsp.contLen = 0;
@@ -123,17 +129,36 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 
     int32_t code = vnodeProcessWriteReq(pVnode->pImpl, pRpc, version++, &rsp);
     tmsgSendRsp(&rsp);
+  }
+#else
+  // sync integration response
+  for (int i = 0; i < taosArrayGetSize(pArray); i++) {
+    SNodeMsg *pMsg;
+    SRpcMsg  *pRpc;
 
-#if 0
-    if (pRsp != NULL) {
-      pRsp->ahandle = pRpc->ahandle;
-      taosMemoryFree(pRsp);
+    pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
+    pRpc = &pMsg->rpcMsg;
+
+    rsp.ahandle = pRpc->ahandle;
+    rsp.handle = pRpc->handle;
+    rsp.pCont = NULL;
+    rsp.contLen = 0;
+
+    int32_t ret = syncPropose(vnodeGetSyncHandle(pVnode->pImpl), pRpc, false);
+    if (ret == TAOS_SYNC_PROPOSE_NOT_LEADER) {
+      rsp.code = TSDB_CODE_SYN_NOT_LEADER;
+      tmsgSendRsp(&rsp);
+    } else if (ret == TAOS_SYNC_PROPOSE_OTHER_ERROR) {
+      rsp.code = TSDB_CODE_SYN_INTERNAL_ERROR;
+      tmsgSendRsp(&rsp);
+    } else if (ret == TAOS_SYNC_PROPOSE_SUCCESS) {
+      // ok
+      // send response in applyQ
     } else {
-      if (code != 0 && terrno != 0) code = terrno;
-      vmSendRsp(pVnode->pWrapper, pMsg, code);
+      assert(0);
     }
-#endif
   }
+#endif
 
   for (int32_t i = 0; i < numOfMsgs; i++) {
     SNodeMsg *pMsg = *(SNodeMsg **)taosArrayGet(pArray, i);
@@ -148,13 +173,44 @@ static void vmProcessWriteQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO
 static void vmProcessApplyQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) {
   SVnodeObj *pVnode = pInfo->ahandle;
   SNodeMsg  *pMsg = NULL;
+  SRpcMsg    rsp;
+
+  // static int64_t version = 0;
 
   for (int32_t i = 0; i < numOfMsgs; ++i) {
+#if 1
+    // sync integration
+
     taosGetQitem(qall, (void **)&pMsg);
 
-    // todo
-    SRpcMsg *pRsp = NULL;
-    // (void)vnodeProcessWriteReq(pVnode->pImpl, &pMsg->rpcMsg, &pRsp);
+    // init response rpc msg
+    rsp.code = 0;
+    rsp.pCont = NULL;
+    rsp.contLen = 0;
+
+    // get original rpc msg
+    assert(pMsg->rpcMsg.msgType == TDMT_VND_SYNC_APPLY_MSG);
+    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgFromRpcMsg2(&pMsg->rpcMsg);
+    syncApplyMsgLog2("==vmProcessApplyQueue==", pSyncApplyMsg);
+    SRpcMsg originalRpcMsg;
+    syncApplyMsg2OriginalRpcMsg(pSyncApplyMsg, &originalRpcMsg);
+
+    // apply data into tsdb
+    if (vnodeProcessWriteReq(pVnode->pImpl, &originalRpcMsg, pSyncApplyMsg->fsmMeta.index, &rsp) < 0) {
+      rsp.code = terrno;
+      dTrace("vnodeProcessWriteReq error, code:%d", terrno);
+    }
+
+    syncApplyMsgDestroy(pSyncApplyMsg);
+    rpcFreeCont(originalRpcMsg.pCont);
+
+    // if leader, send response
+    if (pMsg->rpcMsg.handle != NULL && pMsg->rpcMsg.ahandle != NULL) {
+      rsp.ahandle = pMsg->rpcMsg.ahandle;
+      rsp.handle = pMsg->rpcMsg.handle;
+      tmsgSendRsp(&rsp);
+    }
+#endif
   }
 }
 
@@ -307,6 +363,10 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueT
         dTrace("msg:%p, will be put into vnode-merge queue", pMsg);
         taosWriteQitem(pVnode->pMergeQ, pMsg);
         break;
+      case SYNC_QUEUE:  // sync integration
+        dTrace("msg:%p, will be put into vnode-sync queue", pMsg);
+        taosWriteQitem(pVnode->pSyncQ, pMsg);
+        break;
       default:
         code = -1;
         terrno = TSDB_CODE_INVALID_PARA;
@@ -333,6 +393,11 @@ int32_t vmPutMsgToMergeQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
   return vmPutRpcMsgToQueue(pWrapper, pRpc, MERGE_QUEUE);
 }
 
+// sync integration
+int32_t vmPutMsgToSyncQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) {
+  return vmPutRpcMsgToQueue(pWrapper, pRpc, SYNC_QUEUE);
+}
+
 int32_t vmGetQueueSize(SMgmtWrapper *pWrapper, int32_t vgId, EQueueType qtype) {
   int32_t    size = -1;
   SVnodeObj *pVnode = vmAcquireVnode(pWrapper->pMgmt, vgId);

source/dnode/mnode/impl/src/mndDnode.c

@@ -52,9 +52,9 @@ static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq);
 static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp);
 static int32_t mndProcessStatusReq(SNodeMsg *pReq);
 
-static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextConfig(SMnode *pMnode, void *pIter);
-static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows);
+static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
 static void    mndCancelGetNextDnode(SMnode *pMnode, void *pIter);
 
 int32_t mndInitDnode(SMnode *pMnode) {
@@ -335,10 +335,13 @@ static int32_t mndProcessStatusReq(SNodeMsg *pReq) {
       }
       bool roleChanged = false;
       for (int32_t vg = 0; vg < pVgroup->replica; ++vg) {
-        if (pVgroup->vnodeGid[vg].role != pVload->syncState) {
-          roleChanged = true;
+        // sync integration
+        if (pVgroup->vnodeGid[vg].dnodeId == statusReq.dnodeId) {
+          if (pVgroup->vnodeGid[vg].role != pVload->syncState) {
+            roleChanged = true;
+          }
+          pVgroup->vnodeGid[vg].role = pVload->syncState;
         }
-        pVgroup->vnodeGid[vg].role = pVload->syncState;
       }
       if (roleChanged) {
         // notify scheduler role has changed
@@ -634,7 +637,7 @@ static int32_t mndProcessConfigDnodeRsp(SNodeMsg *pRsp) {
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode *pMnode = pReq->pNode;
   int32_t totalRows = 0;
   int32_t numOfRows = 0;
@@ -667,12 +670,12 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock*
 
     STR_WITH_MAXSIZE_TO_VARSTR(buf, cfgOpts[i], TSDB_CONFIG_OPTION_LEN);
 
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) buf, false);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)buf, false);
 
     STR_WITH_MAXSIZE_TO_VARSTR(bufVal, cfgVals[i], TSDB_CONIIG_VALUE_LEN);
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) bufVal, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)bufVal, false);
 
     numOfRows++;
   }
@@ -683,7 +686,7 @@ static int32_t mndRetrieveConfigs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock*
 
 static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter) {}
 
-static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) {
+static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) {
   SMnode    *pMnode = pReq->pNode;
   SSdb      *pSdb = pMnode->pSdb;
   int32_t    numOfRows = 0;
@@ -698,8 +701,8 @@ static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
 
     cols = 0;
 
-    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) &pDnode->id, false);
+    SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->id, false);
 
     char buf[tListLen(pDnode->ep) + VARSTR_HEADER_SIZE] = {0};
     STR_WITH_MAXSIZE_TO_VARSTR(buf, pDnode->ep, pShow->bytes[cols]);
@@ -709,19 +712,19 @@ static int32_t mndRetrieveDnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* p
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     int16_t id = mndGetVnodesNum(pMnode, pDnode->id);
-    colDataAppend(pColInfo, numOfRows, (const char*) &id, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)&id, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->numOfSupportVnodes, false);
 
     char b1[9] = {0};
-    STR_TO_VARSTR(b1, online? "ready":"offline");
+    STR_TO_VARSTR(b1, online ? "ready" : "offline");
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, b1, false);
 
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
-    colDataAppend(pColInfo, numOfRows, (const char*) &pDnode->createdTime, false);
+    colDataAppend(pColInfo, numOfRows, (const char *)&pDnode->createdTime, false);
 
     char b[tListLen(offlineReason) + VARSTR_HEADER_SIZE] = {0};
     STR_TO_VARSTR(b, online ? "" : offlineReason[pDnode->offlineReason]);

source/dnode/vnode/CMakeLists.txt

@@ -15,6 +15,7 @@ target_sources(
     "src/vnd/vnodeStateMgr.c"
     "src/vnd/vnodeModule.c"
     "src/vnd/vnodeSvr.c"
+    "src/vnd/vnodeSync.c"
 
     # meta
     "src/meta/metaOpen.c"

source/dnode/vnode/inc/vnode.h

@@ -21,6 +21,7 @@
 #include "tqueue.h"
 #include "trpc.h"
 
+#include "sync.h"
 #include "tarray.h"
 #include "tfs.h"
 #include "wal.h"
@@ -60,6 +61,9 @@ int32_t vnodeSync(SVnode *pVnode);
 int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad);
 int     vnodeValidateTableHash(SVnodeCfg *pVnodeOptions, char *tableFName);
 
+int64_t vnodeGetSyncHandle(SVnode *pVnode);
+void    vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot);
+
 // meta
 typedef struct SMeta      SMeta;  // todo: remove
 typedef struct SMTbCursor SMTbCursor;
@@ -147,6 +151,7 @@ struct SVnodeCfg {
   bool     isWeak;
   STsdbCfg tsdbCfg;
   SWalCfg  walCfg;
+  SSyncCfg syncCfg;  // sync integration
   uint32_t hashBegin;
   uint32_t hashEnd;
   int8_t   hashMethod;
@@ -166,6 +171,11 @@ typedef struct {
   uint64_t uid;
 } STableKeyInfo;
 
+// sync integration
+void    vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
+void    vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
+int32_t vnodeSyncStart(SVnode *pVnode);
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/vnodeInt.h

@@ -68,13 +68,13 @@ typedef struct {
 } SStreamSinkInfo;
 
 typedef struct {
-  SVnode*   pVnode;
-  SHashObj* pHash;  // streamId -> SStreamSinkInfo
+  SVnode   *pVnode;
+  SHashObj *pHash;  // streamId -> SStreamSinkInfo
 } SSink;
 
 // SVState
 struct SVState {
-  int64_t processed;
+  // int64_t processed;
   int64_t committed;
   int64_t applied;
 };
@@ -85,25 +85,26 @@ struct SVnodeInfo {
 };
 
 struct SVnode {
-  char*      path;
+  char      *path;
   SVnodeCfg  config;
   SVState    state;
-  STfs*      pTfs;
+  STfs      *pTfs;
   SMsgCb     msgCb;
-  SVBufPool* pBufPool;
-  SMeta*     pMeta;
-  STsdb*     pTsdb;
-  SWal*      pWal;
-  STQ*       pTq;
-  SSink*     pSink;
+  SVBufPool *pBufPool;
+  SMeta     *pMeta;
+  STsdb     *pTsdb;
+  SWal      *pWal;
+  STQ       *pTq;
+  SSink     *pSink;
+  int64_t    sync;  // sync integration
   tsem_t     canCommit;
-  SQHandle*  pQuery;
+  SQHandle  *pQuery;
 };
 
 #define TD_VID(PVNODE) (PVNODE)->config.vgId
 
 // sma
-void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
+void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data);
 
 #include "vnd.h"
 
@@ -113,6 +114,8 @@ void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data);
 
 #include "tq.h"
 
+#include "vnodeSync.h"
+
 #ifdef __cplusplus
 }
 #endif

source/dnode/vnode/src/inc/vnodeSync.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_VNODE_SYNC_H_
+#define _TD_VNODE_SYNC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int32_t vnodeSyncOpen(SVnode *pVnode, char *path);
+int32_t vnodeSyncStart(SVnode *pVnode);
+void    vnodeSyncClose(SVnode *pVnode);
+
+void vnodeSyncSetQ(SVnode *pVnode, void *qHandle);
+void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle);
+
+int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg);
+int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg);
+
+void    vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+void    vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+void    vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta);
+int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot);
+
+SSyncFSM *syncVnodeMakeFsm();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_VNODE_SYNC_H_*/

source/dnode/vnode/src/vnd/vnodeCfg.c

@@ -82,6 +82,18 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
   if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
   if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
 
+  // sync integration
+  if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1;
+  SJson *pNodeInfoArr = tjsonCreateArray();
+  tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", pNodeInfoArr);
+  for (int i = 0; i < pCfg->syncCfg.replicaNum; ++i) {
+    SJson *pNodeInfo = tjsonCreateObject();
+    tjsonAddIntegerToObject(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort);
+    tjsonAddStringToObject(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn);
+    tjsonAddItemToArray(pNodeInfoArr, pNodeInfo);
+  }
+
   return 0;
 }
 
@@ -120,6 +132,21 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
   if (tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
   if (tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
 
+  // sync integration
+  if (tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1;
+
+  SJson *pNodeInfoArr = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo");
+  int    arraySize = tjsonGetArraySize(pNodeInfoArr);
+  assert(arraySize == pCfg->syncCfg.replicaNum);
+
+  for (int i = 0; i < arraySize; ++i) {
+    SJson *pNodeInfo = tjsonGetArrayItem(pNodeInfoArr, i);
+    assert(pNodeInfo != NULL);
+    tjsonGetNumberValue(pNodeInfo, "nodePort", (pCfg->syncCfg.nodeInfo)[i].nodePort);
+    tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn);
+  }
+
   return 0;
 }
 

source/dnode/vnode/src/vnd/vnodeCommit.c

@@ -158,12 +158,23 @@ int vnodeSyncCommit(SVnode *pVnode) {
 }
 
 static int vnodeCommit(void *arg) {
-  SVnode *pVnode = (SVnode *)arg;
+  SVnode    *pVnode = (SVnode *)arg;
+  char       dir[TSDB_FILENAME_LEN];
+  SVnodeInfo info = {0};
+
+  snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path);
+  info.config = pVnode->config;
+  info.state.committed = pVnode->state.applied;
+  info.state.applied = pVnode->state.applied;
+
+  vnodeSaveInfo(dir, &info);
 
   // metaCommit(pVnode->pMeta);
   tqCommit(pVnode->pTq);
   tsdbCommit(pVnode->pTsdb);
 
+  vnodeCommitInfo(dir, &info);
+
   vnodeBufPoolRecycle(pVnode);
   tsem_post(&(pVnode->canCommit));
   return 0;
@@ -185,6 +196,7 @@ static int vnodeEncodeState(const void *pObj, SJson *pJson) {
   const SVState *pState = (SVState *)pObj;
 
   if (tjsonAddIntegerToObject(pJson, "commit version", pState->committed) < 0) return -1;
+  if (tjsonAddIntegerToObject(pJson, "applied version", pState->applied) < 0) return -1;
 
   return 0;
 }
@@ -193,6 +205,7 @@ static int vnodeDecodeState(const SJson *pJson, void *pObj) {
   SVState *pState = (SVState *)pObj;
 
   if (tjsonGetNumberValue(pJson, "commit version", pState->committed) < 0) return -1;
+  if (tjsonGetNumberValue(pJson, "applied version", pState->applied) < 0) return -1;
 
   return 0;
 }

source/dnode/vnode/src/vnd/vnodeOpen.c

@@ -35,6 +35,8 @@ int vnodeCreate(const char *path, SVnodeCfg *pCfg, STfs *pTfs) {
 
   snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
   info.config = *pCfg;
+  info.state.committed = -1;
+  info.state.applied = -1;
 
   if (vnodeSaveInfo(dir, &info) < 0 || vnodeCommitInfo(dir, &info) < 0) {
     vError("vgId: %d failed to save vnode config since %s", pCfg->vgId, tstrerror(terrno));
@@ -75,8 +77,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
   pVnode->path = (char *)&pVnode[1];
   strcpy(pVnode->path, path);
   pVnode->config = info.config;
-  pVnode->state.committed = info.state.committed;
-  pVnode->state.processed = pVnode->state.applied = pVnode->state.committed;
+  pVnode->state = info.state;
   pVnode->pTfs = pTfs;
   pVnode->msgCb = msgCb;
 
@@ -124,6 +125,12 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
     goto _err;
   }
 
+  // sync integration
+  // open sync
+  if (vnodeSyncOpen(pVnode, dir)) {
+    goto _err;
+  }
+
 #if 0
   if (vnodeBegin() < 0) {
     goto _err;
@@ -149,6 +156,10 @@ void vnodeClose(SVnode *pVnode) {
     vnodeSyncCommit(pVnode);
     // close vnode
     vnodeQueryClose(pVnode);
+
+    // sync integration
+    vnodeSyncClose(pVnode);
+
     walClose(pVnode->pWal);
     tqClose(pVnode->pTq);
     tsdbClose(pVnode->pTsdb);
@@ -159,3 +170,7 @@ void vnodeClose(SVnode *pVnode) {
     taosMemoryFree(pVnode);
   }
 }
+
+int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; }
+
+void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot) { pSnapshot->lastApplyIndex = pVnode->state.committed; }
\ No newline at end of file

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -13,6 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "sync.h"
+#include "syncTools.h"
 #include "vnodeInt.h"
 
 static int vnodeProcessCreateStbReq(SVnode *pVnode, void *pReq);
@@ -21,6 +23,7 @@ static int vnodeProcessAlterStbReq(SVnode *pVnode, void *pReq);
 static int vnodeProcessSubmitReq(SVnode *pVnode, SSubmitReq *pSubmitReq, SRpcMsg *pRsp);
 
 int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
+#if 0
   SNodeMsg *pMsg;
   SRpcMsg  *pRpc;
 
@@ -38,6 +41,7 @@ int vnodePreprocessWriteReqs(SVnode *pVnode, SArray *pMsgs, int64_t *version) {
 
   walFsync(pVnode->pWal, false);
 
+#endif
   return 0;
 }
 
@@ -202,8 +206,86 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) {
   tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data);
 }
 
+// sync integration
 int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
-  /*vInfo("sync message is processed");*/
+  SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync);
+  assert(pSyncNode != NULL);
+
+  ESyncState state = syncGetMyRole(pVnode->sync);
+  SyncTerm   currentTerm = syncGetMyTerm(pVnode->sync);
+
+  SMsgHead *pHead = pMsg->pCont;
+
+  char  logBuf[512];
+  char *syncNodeStr = sync2SimpleStr(pVnode->sync);
+  snprintf(logBuf, sizeof(logBuf), "==vnodeProcessSyncReq== msgType:%d, syncNode: %s", pMsg->msgType, syncNodeStr);
+  syncRpcMsgLog2(logBuf, pMsg);
+  taosMemoryFree(syncNodeStr);
+
+  SRpcMsg *pRpcMsg = pMsg;
+
+  if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) {
+    SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnTimeoutCb(pSyncNode, pSyncMsg);
+    syncTimeoutDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) {
+    SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnPingCb(pSyncNode, pSyncMsg);
+    syncPingDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) {
+    SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnPingReplyCb(pSyncNode, pSyncMsg);
+    syncPingReplyDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) {
+    SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnClientRequestCb(pSyncNode, pSyncMsg);
+    syncClientRequestDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) {
+    SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnRequestVoteCb(pSyncNode, pSyncMsg);
+    syncRequestVoteDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) {
+    SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnRequestVoteReplyCb(pSyncNode, pSyncMsg);
+    syncRequestVoteReplyDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) {
+    SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnAppendEntriesCb(pSyncNode, pSyncMsg);
+    syncAppendEntriesDestroy(pSyncMsg);
+
+  } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) {
+    SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg);
+    assert(pSyncMsg != NULL);
+
+    syncNodeOnAppendEntriesReplyCb(pSyncNode, pSyncMsg);
+    syncAppendEntriesReplyDestroy(pSyncMsg);
+
+  } else {
+    vError("==vnodeProcessSyncReq== error msg type:%d", pRpcMsg->msgType);
+  }
+
+  syncNodeRelease(pSyncNode);
+
   return 0;
 }
 

source/dnode/vnode/src/vnd/vnodeSync.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "sync.h"
+#include "syncTools.h"
+#include "tmsgcb.h"
+#include "vnodeInt.h"
+
+// sync integration
+
+int32_t vnodeSyncOpen(SVnode *pVnode, char *path) {
+  SSyncInfo syncInfo;
+  syncInfo.vgId = pVnode->config.vgId;
+  SSyncCfg *pCfg = &(syncInfo.syncCfg);
+  pCfg->replicaNum = pVnode->config.syncCfg.replicaNum;
+  pCfg->myIndex = pVnode->config.syncCfg.myIndex;
+  memcpy(pCfg->nodeInfo, pVnode->config.syncCfg.nodeInfo, sizeof(pCfg->nodeInfo));
+
+  snprintf(syncInfo.path, sizeof(syncInfo.path), "%s/sync", path);
+  syncInfo.pWal = pVnode->pWal;
+
+  syncInfo.pFsm = syncVnodeMakeFsm(pVnode);
+  syncInfo.rpcClient = NULL;
+  syncInfo.FpSendMsg = vnodeSendMsg;
+  syncInfo.queue = NULL;
+  syncInfo.FpEqMsg = vnodeSyncEqMsg;
+
+  pVnode->sync = syncOpen(&syncInfo);
+  assert(pVnode->sync > 0);
+
+  // for test
+  setPingTimerMS(pVnode->sync, 3000);
+  setElectTimerMS(pVnode->sync, 500);
+  setHeartbeatTimerMS(pVnode->sync, 100);
+
+  return 0;
+}
+
+int32_t vnodeSyncStart(SVnode *pVnode) {
+  syncStart(pVnode->sync);
+  return 0;
+}
+
+void vnodeSyncClose(SVnode *pVnode) {
+  // stop by ref id
+  syncStop(pVnode->sync);
+}
+
+void vnodeSyncSetQ(SVnode *pVnode, void *qHandle) { syncSetQ(pVnode->sync, (void *)(&(pVnode->msgCb))); }
+
+void vnodeSyncSetRpc(SVnode *pVnode, void *rpcHandle) { syncSetRpc(pVnode->sync, (void *)(&(pVnode->msgCb))); }
+
+int32_t vnodeSyncEqMsg(void *qHandle, SRpcMsg *pMsg) {
+  int32_t ret = 0;
+  SMsgCb *pMsgCb = qHandle;
+  if (pMsgCb->queueFps[SYNC_QUEUE] != NULL) {
+    tmsgPutToQueue(qHandle, SYNC_QUEUE, pMsg);
+  } else {
+    vError("vnodeSyncEqMsg queue is NULL, SYNC_QUEUE:%d", SYNC_QUEUE);
+  }
+  return ret;
+}
+
+int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg) {
+  int32_t ret = 0;
+  SMsgCb *pMsgCb = rpcHandle;
+  if (pMsgCb->queueFps[SYNC_QUEUE] != NULL) {
+    tmsgSendReq(rpcHandle, pEpSet, pMsg);
+  } else {
+    vError("vnodeSendMsg queue is NULL, SYNC_QUEUE:%d", SYNC_QUEUE);
+  }
+  return ret;
+}
+
+int32_t vnodeSyncGetSnapshotCb(struct SSyncFSM *pFsm, SSnapshot *pSnapshot) {
+  SVnode *pVnode = (SVnode *)(pFsm->data);
+  vnodeGetSnapshot(pVnode, pSnapshot);
+
+  /*
+  pSnapshot->data = NULL;
+  pSnapshot->lastApplyIndex = 0;
+  pSnapshot->lastApplyTerm = 0;
+  */
+
+  return 0;
+}
+
+void vnodeSyncCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+  SyncIndex beginIndex = SYNC_INDEX_INVALID;
+  if (pFsm->FpGetSnapshot != NULL) {
+    SSnapshot snapshot;
+    pFsm->FpGetSnapshot(pFsm, &snapshot);
+    beginIndex = snapshot.lastApplyIndex;
+  }
+
+  if (cbMeta.index > beginIndex) {
+    char logBuf[256];
+    snprintf(
+        logBuf, sizeof(logBuf),
+        "==callback== ==CommitCb== execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, beginIndex :%ld\n",
+        pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state), beginIndex);
+    syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+
+    SVnode       *pVnode = (SVnode *)(pFsm->data);
+    SyncApplyMsg *pSyncApplyMsg = syncApplyMsgBuild2(pMsg, pVnode->config.vgId, &cbMeta);
+    SRpcMsg       applyMsg;
+    syncApplyMsg2RpcMsg(pSyncApplyMsg, &applyMsg);
+    syncApplyMsgDestroy(pSyncApplyMsg);
+
+    /*
+        SRpcMsg applyMsg;
+        applyMsg = *pMsg;
+        applyMsg.pCont = rpcMallocCont(applyMsg.contLen);
+        assert(applyMsg.contLen == pMsg->contLen);
+        memcpy(applyMsg.pCont, pMsg->pCont, applyMsg.contLen);
+    */
+
+    // recover handle for response
+    SRpcMsg saveRpcMsg;
+    int32_t ret = syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &saveRpcMsg);
+    if (ret == 1 && cbMeta.state == TAOS_SYNC_STATE_LEADER) {
+      applyMsg.handle = saveRpcMsg.handle;
+      applyMsg.ahandle = saveRpcMsg.ahandle;
+    } else {
+      applyMsg.handle = NULL;
+      applyMsg.ahandle = NULL;
+    }
+
+    // put to applyQ
+    tmsgPutToQueue(&(pVnode->msgCb), APPLY_QUEUE, &applyMsg);
+
+  } else {
+    char logBuf[256];
+    snprintf(logBuf, sizeof(logBuf),
+             "==callback== ==CommitCb== do not execute, pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s, "
+             "beginIndex :%ld\n",
+             pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state),
+             beginIndex);
+    syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+  }
+}
+
+void vnodeSyncPreCommitCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf),
+           "==callback== ==PreCommitCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n", pFsm, cbMeta.index,
+           cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+  syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+}
+
+void vnodeSyncRollBackCb(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbMeta) {
+  char logBuf[256];
+  snprintf(logBuf, sizeof(logBuf), "==callback== ==RollBackCb== pFsm:%p, index:%ld, isWeak:%d, code:%d, state:%d %s \n",
+           pFsm, cbMeta.index, cbMeta.isWeak, cbMeta.code, cbMeta.state, syncUtilState2String(cbMeta.state));
+  syncRpcMsgLog2(logBuf, (SRpcMsg *)pMsg);
+}
+
+SSyncFSM *syncVnodeMakeFsm(SVnode *pVnode) {
+  SSyncFSM *pFsm = (SSyncFSM *)taosMemoryMalloc(sizeof(SSyncFSM));
+  pFsm->data = pVnode;
+  pFsm->FpCommitCb = vnodeSyncCommitCb;
+  pFsm->FpPreCommitCb = vnodeSyncPreCommitCb;
+  pFsm->FpRollBackCb = vnodeSyncRollBackCb;
+  pFsm->FpGetSnapshot = vnodeSyncGetSnapshotCb;
+  return pFsm;
+}

source/libs/sync/CMakeLists.txt

@@ -11,7 +11,7 @@ target_link_libraries(
 
 target_include_directories(
     sync
-    PUBLIC "${TD_SOURCE_DIR}/include/libs/sync"
+    PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync"
     PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
 )
 

source/libs/sync/inc/syncEnv.h

@@ -31,10 +31,10 @@ extern "C" {
 #define TIMER_MAX_MS 0x7FFFFFFF
 #define ENV_TICK_TIMER_MS 1000
 #define PING_TIMER_MS 1000
-#define ELECT_TIMER_MS_MIN 150
+#define ELECT_TIMER_MS_MIN 500
 #define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2)
 #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN)
-#define HEARTBEAT_TIMER_MS 30
+#define HEARTBEAT_TIMER_MS 100
 
 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0})
 

source/libs/sync/inc/syncInt.h

@@ -25,6 +25,7 @@ extern "C" {
 #include <stdlib.h>
 #include "cJSON.h"
 #include "sync.h"
+#include "syncTools.h"
 #include "taosdef.h"
 #include "tglobal.h"
 #include "tlog.h"
@@ -67,6 +68,43 @@ extern "C" {
     }                                                                   \
   }
 
+#define sFatalLong(...)                                                 \
+  {                                                                     \
+    if (sDebugFlag & DEBUG_FATAL) {                                     \
+      taosPrintLongString("SYN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); \
+    }                                                                   \
+  }
+#define sErrorLong(...)                                                 \
+  {                                                                     \
+    if (sDebugFlag & DEBUG_ERROR) {                                     \
+      taosPrintLongString("SYN ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); \
+    }                                                                   \
+  }
+#define sWarnLong(...)                                                \
+  {                                                                   \
+    if (sDebugFlag & DEBUG_WARN) {                                    \
+      taosPrintLongString("SYN WARN ", DEBUG_WARN, 255, __VA_ARGS__); \
+    }                                                                 \
+  }
+#define sInfoLong(...)                                                \
+  {                                                                   \
+    if (sDebugFlag & DEBUG_INFO) {                                    \
+      taosPrintLongString("SYN INFO ", DEBUG_INFO, 255, __VA_ARGS__); \
+    }                                                                 \
+  }
+#define sDebugLong(...)                                                        \
+  {                                                                            \
+    if (sDebugFlag & DEBUG_DEBUG) {                                            \
+      taosPrintLongString("SYN DEBUG ", DEBUG_DEBUG, sDebugFlag, __VA_ARGS__); \
+    }                                                                          \
+  }
+#define sTraceLong(...)                                                        \
+  {                                                                            \
+    if (sDebugFlag & DEBUG_TRACE) {                                            \
+      taosPrintLongString("SYN TRACE ", DEBUG_TRACE, sDebugFlag, __VA_ARGS__); \
+    }                                                                          \
+  }
+
 struct SyncTimeout;
 typedef struct SyncTimeout SyncTimeout;
 
@@ -106,17 +144,19 @@ typedef struct SVotesRespond SVotesRespond;
 struct SSyncIndexMgr;
 typedef struct SSyncIndexMgr SSyncIndexMgr;
 
-typedef struct SRaftId {
-  SyncNodeId  addr;  // typedef uint64_t SyncNodeId;
-  SyncGroupId vgId;  // typedef int32_t  SyncGroupId;
-} SRaftId;
+struct SRaftCfg;
+typedef struct SRaftCfg SRaftCfg;
+
+struct SSyncRespMgr;
+typedef struct SSyncRespMgr SSyncRespMgr;
 
 typedef struct SSyncNode {
   // init by SSyncInfo
   SyncGroupId vgId;
-  SSyncCfg    syncCfg;
+  SRaftCfg*   pRaftCfg;
   char        path[TSDB_FILENAME_LEN];
   char        raftStorePath[TSDB_FILENAME_LEN * 2];
+  char        configPath[TSDB_FILENAME_LEN * 2];
 
   // sync io
   SWal* pWal;
@@ -199,10 +239,14 @@ typedef struct SSyncNode {
   int32_t (*FpOnAppendEntriesReply)(SSyncNode* ths, SyncAppendEntriesReply* pMsg);
   int32_t (*FpOnTimeout)(SSyncNode* pSyncNode, SyncTimeout* pMsg);
 
+  // tools
+  SSyncRespMgr* pSyncRespMgr;
+
 } SSyncNode;
 
 // open/close --------------
 SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo);
+void       syncNodeStart(SSyncNode* pSyncNode);
 void       syncNodeClose(SSyncNode* pSyncNode);
 
 // ping --------------
@@ -222,10 +266,12 @@ int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode);
 int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode);
 
 // utils --------------
-int32_t    syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
-int32_t    syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
-cJSON*     syncNode2Json(const SSyncNode* pSyncNode);
-char*      syncNode2Str(const SSyncNode* pSyncNode);
+int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg);
+int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg);
+cJSON*  syncNode2Json(const SSyncNode* pSyncNode);
+char*   syncNode2Str(const SSyncNode* pSyncNode);
+char*   syncNode2SimpleStr(const SSyncNode* pSyncNode);
+
 SSyncNode* syncNodeAcquire(int64_t rid);
 void       syncNodeRelease(SSyncNode* pNode);
 

source/libs/sync/inc/syncMessage.h

@@ -27,22 +27,6 @@ extern "C" {
 #include "syncInt.h"
 #include "taosdef.h"
 
-// encode as uint32
-typedef enum ESyncMessageType {
-  SYNC_UNKNOWN = 9999,
-  SYNC_TIMEOUT = 99,
-  SYNC_PING = 101,
-  SYNC_PING_REPLY = 103,
-  SYNC_CLIENT_REQUEST = 105,
-  SYNC_CLIENT_REQUEST_REPLY = 107,
-  SYNC_REQUEST_VOTE = 109,
-  SYNC_REQUEST_VOTE_REPLY = 111,
-  SYNC_APPEND_ENTRIES = 113,
-  SYNC_APPEND_ENTRIES_REPLY = 115,
-  SYNC_RESPONSE = 119,
-
-} ESyncMessageType;
-
 // ---------------------------------------------
 cJSON* syncRpcMsg2Json(SRpcMsg* pRpcMsg);
 cJSON* syncRpcUnknownMsg2Json();
@@ -53,263 +37,7 @@ void syncRpcMsgPrint(SRpcMsg* pMsg);
 void syncRpcMsgPrint2(char* s, SRpcMsg* pMsg);
 void syncRpcMsgLog(SRpcMsg* pMsg);
 void syncRpcMsgLog2(char* s, SRpcMsg* pMsg);
-
-// ---------------------------------------------
-typedef enum ESyncTimeoutType {
-  SYNC_TIMEOUT_PING = 100,
-  SYNC_TIMEOUT_ELECTION,
-  SYNC_TIMEOUT_HEARTBEAT,
-} ESyncTimeoutType;
-
-typedef struct SyncTimeout {
-  uint32_t         bytes;
-  uint32_t         msgType;
-  ESyncTimeoutType timeoutType;
-  uint64_t         logicClock;
-  int32_t          timerMS;
-  void*            data;  // need optimized
-} SyncTimeout;
-
-SyncTimeout* syncTimeoutBuild();
-SyncTimeout* syncTimeoutBuild2(ESyncTimeoutType timeoutType, uint64_t logicClock, int32_t timerMS, void* data);
-void         syncTimeoutDestroy(SyncTimeout* pMsg);
-void         syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen);
-void         syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg);
-char*        syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len);  //
-SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len);         //
-void         syncTimeout2RpcMsg(const SyncTimeout* pMsg, SRpcMsg* pRpcMsg);
-void         syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg);
-SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg);  //
-cJSON*       syncTimeout2Json(const SyncTimeout* pMsg);
-char*        syncTimeout2Str(const SyncTimeout* pMsg);  //
-
-// for debug ----------------------
-void syncTimeoutPrint(const SyncTimeout* pMsg);
-void syncTimeoutPrint2(char* s, const SyncTimeout* pMsg);
-void syncTimeoutLog(const SyncTimeout* pMsg);
-void syncTimeoutLog2(char* s, const SyncTimeout* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncPing {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  uint32_t dataLen;
-  char     data[];
-} SyncPing;
-
-SyncPing* syncPingBuild(uint32_t dataLen);
-SyncPing* syncPingBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str);
-SyncPing* syncPingBuild3(const SRaftId* srcId, const SRaftId* destId);
-void      syncPingDestroy(SyncPing* pMsg);
-void      syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen);
-void      syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg);
-char*     syncPingSerialize2(const SyncPing* pMsg, uint32_t* len);
-SyncPing* syncPingDeserialize2(const char* buf, uint32_t len);
-void      syncPing2RpcMsg(const SyncPing* pMsg, SRpcMsg* pRpcMsg);
-void      syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg);
-SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg);
-cJSON*    syncPing2Json(const SyncPing* pMsg);
-char*     syncPing2Str(const SyncPing* pMsg);
-
-// for debug ----------------------
-void syncPingPrint(const SyncPing* pMsg);
-void syncPingPrint2(char* s, const SyncPing* pMsg);
-void syncPingLog(const SyncPing* pMsg);
-void syncPingLog2(char* s, const SyncPing* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncPingReply {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  uint32_t dataLen;
-  char     data[];
-} SyncPingReply;
-
-SyncPingReply* syncPingReplyBuild(uint32_t dataLen);
-SyncPingReply* syncPingReplyBuild2(const SRaftId* srcId, const SRaftId* destId, const char* str);
-SyncPingReply* syncPingReplyBuild3(const SRaftId* srcId, const SRaftId* destId);
-void           syncPingReplyDestroy(SyncPingReply* pMsg);
-void           syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen);
-void           syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg);
-char*          syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len);  //
-SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len);           //
-void           syncPingReply2RpcMsg(const SyncPingReply* pMsg, SRpcMsg* pRpcMsg);
-void           syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg);
-SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);  //
-cJSON*         syncPingReply2Json(const SyncPingReply* pMsg);
-char*          syncPingReply2Str(const SyncPingReply* pMsg);  //
-
-// for debug ----------------------
-void syncPingReplyPrint(const SyncPingReply* pMsg);
-void syncPingReplyPrint2(char* s, const SyncPingReply* pMsg);
-void syncPingReplyLog(const SyncPingReply* pMsg);
-void syncPingReplyLog2(char* s, const SyncPingReply* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncClientRequest {
-  uint32_t bytes;
-  uint32_t msgType;
-  uint32_t originalRpcType;
-  uint64_t seqNum;
-  bool     isWeak;
-  uint32_t dataLen;
-  char     data[];
-} SyncClientRequest;
-
-SyncClientRequest* syncClientRequestBuild(uint32_t dataLen);
-SyncClientRequest* syncClientRequestBuild2(const SRpcMsg* pOriginalRpcMsg, uint64_t seqNum, bool isWeak);  // step 1
-void               syncClientRequestDestroy(SyncClientRequest* pMsg);
-void               syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen);
-void               syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg);
-char*              syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len);
-SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len);
-void               syncClientRequest2RpcMsg(const SyncClientRequest* pMsg, SRpcMsg* pRpcMsg);  // step 2
-void               syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg);
-SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg);  // step 3
-cJSON*             syncClientRequest2Json(const SyncClientRequest* pMsg);
-char*              syncClientRequest2Str(const SyncClientRequest* pMsg);
-
-// for debug ----------------------
-void syncClientRequestPrint(const SyncClientRequest* pMsg);
-void syncClientRequestPrint2(char* s, const SyncClientRequest* pMsg);
-void syncClientRequestLog(const SyncClientRequest* pMsg);
-void syncClientRequestLog2(char* s, const SyncClientRequest* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncClientRequestReply {
-  uint32_t bytes;
-  uint32_t msgType;
-  int32_t  errCode;
-  SRaftId  leaderHint;
-} SyncClientRequestReply;
-
-// ---------------------------------------------
-typedef struct SyncRequestVote {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  SyncTerm  term;
-  SyncIndex lastLogIndex;
-  SyncTerm  lastLogTerm;
-} SyncRequestVote;
-
-SyncRequestVote* syncRequestVoteBuild();
-void             syncRequestVoteDestroy(SyncRequestVote* pMsg);
-void             syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen);
-void             syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg);
-char*            syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len);
-SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len);
-void             syncRequestVote2RpcMsg(const SyncRequestVote* pMsg, SRpcMsg* pRpcMsg);
-void             syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg);
-SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg);
-cJSON*           syncRequestVote2Json(const SyncRequestVote* pMsg);
-char*            syncRequestVote2Str(const SyncRequestVote* pMsg);
-
-// for debug ----------------------
-void syncRequestVotePrint(const SyncRequestVote* pMsg);
-void syncRequestVotePrint2(char* s, const SyncRequestVote* pMsg);
-void syncRequestVoteLog(const SyncRequestVote* pMsg);
-void syncRequestVoteLog2(char* s, const SyncRequestVote* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncRequestVoteReply {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  SyncTerm term;
-  bool     voteGranted;
-} SyncRequestVoteReply;
-
-SyncRequestVoteReply* syncRequestVoteReplyBuild();
-void                  syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg);
-void                  syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen);
-void                  syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg);
-char*                 syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len);
-SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len);
-void                  syncRequestVoteReply2RpcMsg(const SyncRequestVoteReply* pMsg, SRpcMsg* pRpcMsg);
-void                  syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply* pMsg);
-SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
-cJSON*                syncRequestVoteReply2Json(const SyncRequestVoteReply* pMsg);
-char*                 syncRequestVoteReply2Str(const SyncRequestVoteReply* pMsg);
-
-// for debug ----------------------
-void syncRequestVoteReplyPrint(const SyncRequestVoteReply* pMsg);
-void syncRequestVoteReplyPrint2(char* s, const SyncRequestVoteReply* pMsg);
-void syncRequestVoteReplyLog(const SyncRequestVoteReply* pMsg);
-void syncRequestVoteReplyLog2(char* s, const SyncRequestVoteReply* pMsg);
-
 // ---------------------------------------------
-typedef struct SyncAppendEntries {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  SyncTerm  term;
-  SyncIndex prevLogIndex;
-  SyncTerm  prevLogTerm;
-  SyncIndex commitIndex;
-  uint32_t  dataLen;
-  char      data[];
-} SyncAppendEntries;
-
-SyncAppendEntries* syncAppendEntriesBuild(uint32_t dataLen);
-void               syncAppendEntriesDestroy(SyncAppendEntries* pMsg);
-void               syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen);
-void               syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg);
-char*              syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len);
-SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len);
-void               syncAppendEntries2RpcMsg(const SyncAppendEntries* pMsg, SRpcMsg* pRpcMsg);
-void               syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg);
-SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg);
-cJSON*             syncAppendEntries2Json(const SyncAppendEntries* pMsg);
-char*              syncAppendEntries2Str(const SyncAppendEntries* pMsg);
-
-// for debug ----------------------
-void syncAppendEntriesPrint(const SyncAppendEntries* pMsg);
-void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg);
-void syncAppendEntriesLog(const SyncAppendEntries* pMsg);
-void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg);
-
-// ---------------------------------------------
-typedef struct SyncAppendEntriesReply {
-  uint32_t bytes;
-  uint32_t msgType;
-  SRaftId  srcId;
-  SRaftId  destId;
-  // private data
-  SyncTerm  term;
-  bool      success;
-  SyncIndex matchIndex;
-} SyncAppendEntriesReply;
-
-SyncAppendEntriesReply* syncAppendEntriesReplyBuild();
-void                    syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg);
-void                    syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen);
-void                    syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg);
-char*                   syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len);
-SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len);
-void                    syncAppendEntriesReply2RpcMsg(const SyncAppendEntriesReply* pMsg, SRpcMsg* pRpcMsg);
-void                    syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesReply* pMsg);
-SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg);
-cJSON*                  syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg);
-char*                   syncAppendEntriesReply2Str(const SyncAppendEntriesReply* pMsg);
-
-// for debug ----------------------
-void syncAppendEntriesReplyPrint(const SyncAppendEntriesReply* pMsg);
-void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg);
-void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg);
-void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg);
 
 #ifdef __cplusplus
 }

source/libs/sync/inc/syncRaftCfg.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RAFT_CFG_H
+#define _TD_LIBS_SYNC_RAFT_CFG_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "cJSON.h"
+#include "syncInt.h"
+#include "taosdef.h"
+
+typedef struct SRaftCfg {
+  SSyncCfg  cfg;
+  TdFilePtr pFile;
+  char      path[TSDB_FILENAME_LEN * 2];
+} SRaftCfg;
+
+SRaftCfg *raftCfgOpen(const char *path);
+int32_t   raftCfgClose(SRaftCfg *pRaftCfg);
+int32_t   raftCfgPersist(SRaftCfg *pRaftCfg);
+
+cJSON * syncCfg2Json(SSyncCfg *pSyncCfg);
+char *  syncCfg2Str(SSyncCfg *pSyncCfg);
+int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg);
+int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg);
+
+cJSON *raftCfg2Json(SRaftCfg *pRaftCfg);
+char * raftCfg2Str(SRaftCfg *pRaftCfg);
+
+int32_t syncCfgCreateFile(SSyncCfg *pCfg, const char *path);
+
+// for debug ----------------------
+void syncCfgPrint(SSyncCfg *pCfg);
+void syncCfgPrint2(char *s, SSyncCfg *pCfg);
+void syncCfgLog(SSyncCfg *pCfg);
+void syncCfgLog2(char *s, SSyncCfg *pCfg);
+
+void raftCfgPrint(SRaftCfg *pCfg);
+void raftCfgPrint2(char *s, SRaftCfg *pCfg);
+void raftCfgLog(SRaftCfg *pCfg);
+void raftCfgLog2(char *s, SRaftCfg *pCfg);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_RAFT_CFG_H*/

source/libs/sync/inc/syncRaftEntry.h

@@ -27,29 +27,22 @@ extern "C" {
 #include "syncMessage.h"
 #include "taosdef.h"
 
-typedef enum EntryType {
-  SYNC_RAFT_ENTRY_NOOP = 0,
-  SYNC_RAFT_ENTRY_DATA = 1,
-  SYNC_RAFT_ENTRY_CONFIG = 2,
-} EntryType;
-
 typedef struct SSyncRaftEntry {
   uint32_t  bytes;
-  uint32_t  msgType;
-  uint32_t  originalRpcType;
+  uint32_t  msgType;          // SyncClientRequest msgType
+  uint32_t  originalRpcType;  // user RpcMsg msgType
   uint64_t  seqNum;
   bool      isWeak;
   SyncTerm  term;
   SyncIndex index;
-  EntryType entryType;
-  uint32_t  dataLen;
-  char      data[];
+  uint32_t  dataLen;  // user RpcMsg.contLen
+  char      data[];   // user RpcMsg.pCont
 } SSyncRaftEntry;
 
 SSyncRaftEntry* syncEntryBuild(uint32_t dataLen);
 SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index);  // step 4
-SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType);
-SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index);
+SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index);
+SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId);
 void            syncEntryDestory(SSyncRaftEntry* pEntry);
 char*           syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len);  // step 5
 SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len);              // step 6

source/libs/sync/inc/syncRaftLog.h

@@ -27,9 +27,6 @@ extern "C" {
 #include "syncRaftEntry.h"
 #include "taosdef.h"
 
-#define SYNC_INDEX_BEGIN 0
-#define SYNC_INDEX_INVALID -1
-
 typedef struct SSyncLogStoreData {
   SSyncNode* pSyncNode;
   SWal*      pWal;

source/libs/sync/inc/syncRaftStore.h

@@ -28,7 +28,7 @@ extern "C" {
 #include "taosdef.h"
 
 #define RAFT_STORE_BLOCK_SIZE 512
-#define RAFT_STORE_PATH_LEN 128
+#define RAFT_STORE_PATH_LEN (TSDB_FILENAME_LEN * 2)
 
 typedef struct SRaftStore {
   SyncTerm  currentTerm;

source/libs/sync/inc/syncRespMgr.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_LIBS_SYNC_RESP_MGR_H
+#define _TD_LIBS_SYNC_RESP_MGR_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include "syncInt.h"
+#include "taosdef.h"
+
+typedef struct SRespStub {
+  SRpcMsg rpcMsg;
+  int64_t createTime;
+} SRespStub;
+
+typedef struct SSyncRespMgr {
+  SHashObj *    pRespHash;
+  int64_t       ttl;
+  void *        data;
+  TdThreadMutex mutex;
+  uint64_t      seqNum;
+} SSyncRespMgr;
+
+SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl);
+void          syncRespMgrDestroy(SSyncRespMgr *pObj);
+int64_t       syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub);
+int32_t       syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index);
+int32_t       syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub);
+int32_t       syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub);
+void          syncRespClean(SSyncRespMgr *pObj);
+void          syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*_TD_LIBS_SYNC_RESP_MGR_H*/

source/libs/sync/inc/syncSnapshot.h

@@ -27,7 +27,6 @@ extern "C" {
 #include "taosdef.h"
 
 int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);
-
 int32_t restoreSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);
 
 #ifdef __cplusplus

source/libs/sync/inc/syncUtil.h

@@ -55,6 +55,8 @@ char*       syncUtilprintBin(char* ptr, uint32_t len);
 char*       syncUtilprintBin2(char* ptr, uint32_t len);
 SyncIndex   syncUtilMinIndex(SyncIndex a, SyncIndex b);
 SyncIndex   syncUtilMaxIndex(SyncIndex a, SyncIndex b);
+void        syncUtilMsgHtoN(void* msg);
+void        syncUtilMsgNtoH(void* msg);
 
 #ifdef __cplusplus
 }

source/libs/sync/src/syncAppendEntries.c

@@ -125,7 +125,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
         "logOK:%d",
         pMsg->term, ths->pRaftStore->currentTerm, ths->state, logOK);
 
-    SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild();
+    SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
     pReply->srcId = ths->myRaftId;
     pReply->destId = pMsg->srcId;
     pReply->term = ths->pRaftStore->currentTerm;
@@ -199,13 +199,20 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
             SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index);
             assert(pRollBackEntry != NULL);
 
-            // maybe is a NOOP ENTRY
-            // assert(pRollBackEntry->entryType == SYNC_RAFT_ENTRY_DATA);
+            if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) {
+              SRpcMsg rpcMsg;
+              syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
+
+              SFsmCbMeta cbMeta;
+              cbMeta.index = pRollBackEntry->index;
+              cbMeta.isWeak = pRollBackEntry->isWeak;
+              cbMeta.code = 0;
+              cbMeta.state = ths->state;
+              cbMeta.seqNum = pRollBackEntry->seqNum;
+              ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta);
+              rpcFreeCont(rpcMsg.pCont);
+            }
 
-            SRpcMsg rpcMsg;
-            syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
-            ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, pRollBackEntry->index, pRollBackEntry->isWeak, 0, ths->state);
-            rpcFreeCont(rpcMsg.pCont);
             syncEntryDestory(pRollBackEntry);
           }
         }
@@ -220,8 +227,14 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
         SRpcMsg rpcMsg;
         syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
         if (ths->pFsm != NULL) {
-          if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
-            ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 2, ths->state);
+          if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+            SFsmCbMeta cbMeta;
+            cbMeta.index = pAppendEntry->index;
+            cbMeta.isWeak = pAppendEntry->isWeak;
+            cbMeta.code = 2;
+            cbMeta.state = ths->state;
+            cbMeta.seqNum = pAppendEntry->seqNum;
+            ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta);
           }
         }
         rpcFreeCont(rpcMsg.pCont);
@@ -245,8 +258,14 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
       SRpcMsg rpcMsg;
       syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
       if (ths->pFsm != NULL) {
-        if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
-          ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, pAppendEntry->index, pAppendEntry->isWeak, 3, ths->state);
+        if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+          SFsmCbMeta cbMeta;
+          cbMeta.index = pAppendEntry->index;
+          cbMeta.isWeak = pAppendEntry->isWeak;
+          cbMeta.code = 3;
+          cbMeta.state = ths->state;
+          cbMeta.seqNum = pAppendEntry->seqNum;
+          ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta);
         }
       }
       rpcFreeCont(rpcMsg.pCont);
@@ -261,7 +280,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
       assert(0);
     }
 
-    SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild();
+    SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId);
     pReply->srcId = ths->myRaftId;
     pReply->destId = pMsg->srcId;
     pReply->term = ths->pRaftStore->currentTerm;
@@ -301,8 +320,14 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
               SRpcMsg rpcMsg;
               syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-              if (ths->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
-                ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, ths->state);
+              if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+                SFsmCbMeta cbMeta;
+                cbMeta.index = pEntry->index;
+                cbMeta.isWeak = pEntry->isWeak;
+                cbMeta.code = 0;
+                cbMeta.state = ths->state;
+                cbMeta.seqNum = pEntry->seqNum;
+                ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta);
               }
 
               rpcFreeCont(rpcMsg.pCont);

source/libs/sync/src/syncAppendEntriesReply.c

@@ -56,6 +56,15 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p
   //    syncNodeUpdateTerm(ths, pMsg->term);
   //  }
 
+  if (pMsg->term > ths->pRaftStore->currentTerm) {
+    char logBuf[128];
+    snprintf(logBuf, sizeof(logBuf), "syncNodeOnAppendEntriesReplyCb error term, receive:%lu current:%lu", pMsg->term,
+             ths->pRaftStore->currentTerm);
+    syncNodeLog2(logBuf, ths);
+    sError("%s", logBuf);
+    return ret;
+  }
+
   assert(pMsg->term == ths->pRaftStore->currentTerm);
 
   if (pMsg->success) {

source/libs/sync/src/syncCommit.c

@@ -97,8 +97,14 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
           SRpcMsg rpcMsg;
           syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-          if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->entryType == SYNC_RAFT_ENTRY_DATA) {
-            pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, pEntry->index, pEntry->isWeak, 0, pSyncNode->state);
+          if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+            SFsmCbMeta cbMeta;
+            cbMeta.index = pEntry->index;
+            cbMeta.isWeak = pEntry->isWeak;
+            cbMeta.code = 0;
+            cbMeta.state = pSyncNode->state;
+            cbMeta.seqNum = pEntry->seqNum;
+            pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, cbMeta);
           }
 
           rpcFreeCont(rpcMsg.pCont);

source/libs/sync/src/syncElection.c

@@ -35,7 +35,7 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) {
 
   int32_t ret = 0;
   for (int i = 0; i < pSyncNode->peersNum; ++i) {
-    SyncRequestVote* pMsg = syncRequestVoteBuild();
+    SyncRequestVote* pMsg = syncRequestVoteBuild(pSyncNode->vgId);
     pMsg->srcId = pSyncNode->myRaftId;
     pMsg->destId = pSyncNode->peersId[i];
     pMsg->term = pSyncNode->pRaftStore->currentTerm;

source/libs/sync/src/syncEnv.c

@@ -31,6 +31,7 @@ int32_t syncEnvStart() {
   taosSeedRand(taosGetTimestampSec());
   gSyncEnv = doSyncEnvStart(gSyncEnv);
   assert(gSyncEnv != NULL);
+  sTrace("syncEnvStart ok!");
   return ret;
 }
 
@@ -91,6 +92,7 @@ static SSyncEnv *doSyncEnvStart() {
 
 static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) {
   taosTmrCleanUp(pSyncEnv->pTimerManager);
+  taosMemoryFree(pSyncEnv);
   return 0;
 }
 

source/libs/sync/src/syncIO.c

@@ -29,7 +29,7 @@ static int32_t  syncIODestroy(SSyncIO *io);
 static int32_t  syncIOStartInternal(SSyncIO *io);
 static int32_t  syncIOStopInternal(SSyncIO *io);
 
-static void   *syncIOConsumerFunc(void *param);
+static void *  syncIOConsumerFunc(void *param);
 static void    syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
 static void    syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet);
 static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey);
@@ -70,9 +70,15 @@ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) {
   assert(pEpSet->numOfEps == 1);
 
   int32_t ret = 0;
-  char    logBuf[256];
-  snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port);
-  syncRpcMsgPrint2(logBuf, pMsg);
+  {
+    syncUtilMsgNtoH(pMsg->pCont);
+
+    char logBuf[256];
+    snprintf(logBuf, sizeof(logBuf), "==syncIOSendMsg== %s:%d", pEpSet->eps[0].fqdn, pEpSet->eps[0].port);
+    syncRpcMsgLog2(logBuf, pMsg);
+
+    syncUtilMsgHtoN(pMsg->pCont);
+  }
 
   pMsg->handle = NULL;
   pMsg->noResp = 1;
@@ -83,7 +89,7 @@ int32_t syncIOSendMsg(void *clientRpc, const SEpSet *pEpSet, SRpcMsg *pMsg) {
 int32_t syncIOEqMsg(void *queue, SRpcMsg *pMsg) {
   int32_t ret = 0;
   char    logBuf[128];
-  syncRpcMsgPrint2((char *)"==syncIOEqMsg==", pMsg);
+  syncRpcMsgLog2((char *)"==syncIOEqMsg==", pMsg);
 
   SRpcMsg *pTemp;
   pTemp = taosAllocateQitem(sizeof(SRpcMsg));
@@ -235,9 +241,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) {
 }
 
 static void *syncIOConsumerFunc(void *param) {
-  SSyncIO   *io = param;
+  SSyncIO *  io = param;
   STaosQall *qall;
-  SRpcMsg   *pRpcMsg, rpcMsg;
+  SRpcMsg *  pRpcMsg, rpcMsg;
   qall = taosAllocateQall();
 
   while (1) {
@@ -252,7 +258,7 @@ static void *syncIOConsumerFunc(void *param) {
       syncRpcMsgLog2((char *)"==syncIOConsumerFunc==", pRpcMsg);
 
       // use switch case instead of if else
-      if (pRpcMsg->msgType == SYNC_PING) {
+      if (pRpcMsg->msgType == TDMT_VND_SYNC_PING) {
         if (io->FpOnSyncPing != NULL) {
           SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -260,7 +266,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncPingDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_PING_REPLY) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_PING_REPLY) {
         if (io->FpOnSyncPingReply != NULL) {
           SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -268,7 +274,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncPingReplyDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_CLIENT_REQUEST) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_CLIENT_REQUEST) {
         if (io->FpOnSyncClientRequest != NULL) {
           SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -276,7 +282,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncClientRequestDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE) {
         if (io->FpOnSyncRequestVote != NULL) {
           SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -284,7 +290,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncRequestVoteDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_REQUEST_VOTE_REPLY) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_REQUEST_VOTE_REPLY) {
         if (io->FpOnSyncRequestVoteReply != NULL) {
           SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -292,7 +298,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncRequestVoteReplyDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES) {
         if (io->FpOnSyncAppendEntries != NULL) {
           SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -300,7 +306,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncAppendEntriesDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_APPEND_ENTRIES_REPLY) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_APPEND_ENTRIES_REPLY) {
         if (io->FpOnSyncAppendEntriesReply != NULL) {
           SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -308,7 +314,7 @@ static void *syncIOConsumerFunc(void *param) {
           syncAppendEntriesReplyDestroy(pSyncMsg);
         }
 
-      } else if (pRpcMsg->msgType == SYNC_TIMEOUT) {
+      } else if (pRpcMsg->msgType == TDMT_VND_SYNC_TIMEOUT) {
         if (io->FpOnSyncTimeout != NULL) {
           SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg);
           assert(pSyncMsg != NULL);
@@ -336,7 +342,7 @@ static void *syncIOConsumerFunc(void *param) {
               rpcMsg.handle = pRpcMsg->handle;
               rpcMsg.code = 0;
 
-              syncRpcMsgPrint2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg);
+              syncRpcMsgLog2((char *)"syncIOConsumerFunc rpcSendResponse --> ", &rpcMsg);
               rpcSendResponse(&rpcMsg);
             }
       */
@@ -350,7 +356,9 @@ static void *syncIOConsumerFunc(void *param) {
 }
 
 static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  syncRpcMsgPrint2((char *)"==syncIOProcessRequest==", pMsg);
+  syncUtilMsgNtoH(pMsg->pCont);
+
+  syncRpcMsgLog2((char *)"==syncIOProcessRequest==", pMsg);
   SSyncIO *io = pParent;
   SRpcMsg *pTemp;
   pTemp = taosAllocateQitem(sizeof(SRpcMsg));
@@ -359,10 +367,10 @@ static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
 }
 
 static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) {
-  if (pMsg->msgType == SYNC_RESPONSE) {
+  if (pMsg->msgType == TDMT_VND_SYNC_COMMON_RESPONSE) {
     sTrace("==syncIOProcessReply==");
   } else {
-    syncRpcMsgPrint2((char *)"==syncIOProcessReply==", pMsg);
+    syncRpcMsgLog2((char *)"==syncIOProcessReply==", pMsg);
   }
   rpcFreeCont(pMsg->pCont);
 }
@@ -408,14 +416,14 @@ static void syncIOTickQ(void *param, void *tmrId) {
   srcId.vgId = -1;
   destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
   destId.vgId = -1;
-  SyncPingReply *pMsg = syncPingReplyBuild2(&srcId, &destId, "syncIOTickQ");
+  SyncPingReply *pMsg = syncPingReplyBuild2(&srcId, &destId, -1, "syncIOTickQ");
 
   SRpcMsg rpcMsg;
   syncPingReply2RpcMsg(pMsg, &rpcMsg);
   SRpcMsg *pTemp;
   pTemp = taosAllocateQitem(sizeof(SRpcMsg));
   memcpy(pTemp, &rpcMsg, sizeof(SRpcMsg));
-  syncRpcMsgPrint2((char *)"==syncIOTickQ==", &rpcMsg);
+  syncRpcMsgLog2((char *)"==syncIOTickQ==", &rpcMsg);
   taosWriteQitem(io->pMsgQ, pTemp);
   syncPingReplyDestroy(pMsg);
 
@@ -430,12 +438,12 @@ static void syncIOTickPing(void *param, void *tmrId) {
   srcId.vgId = -1;
   destId.addr = syncUtilAddr2U64(io->myAddr.eps[0].fqdn, io->myAddr.eps[0].port);
   destId.vgId = -1;
-  SyncPing *pMsg = syncPingBuild2(&srcId, &destId, "syncIOTickPing");
+  SyncPing *pMsg = syncPingBuild2(&srcId, &destId, -1, "syncIOTickPing");
   // SyncPing *pMsg = syncPingBuild3(&srcId, &destId);
 
   SRpcMsg rpcMsg;
   syncPing2RpcMsg(pMsg, &rpcMsg);
-  syncRpcMsgPrint2((char *)"==syncIOTickPing==", &rpcMsg);
+  syncRpcMsgLog2((char *)"==syncIOTickPing==", &rpcMsg);
   rpcSendRequest(io->clientRpc, &io->myAddr, &rpcMsg, NULL);
   syncPingDestroy(pMsg);
 

source/libs/sync/src/syncRaftCfg.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "syncRaftCfg.h"
+#include "cJSON.h"
+#include "syncEnv.h"
+#include "syncUtil.h"
+
+// file must already exist!
+SRaftCfg *raftCfgOpen(const char *path) {
+  SRaftCfg *pCfg = taosMemoryMalloc(sizeof(SRaftCfg));
+  snprintf(pCfg->path, sizeof(pCfg->path), "%s", path);
+
+  pCfg->pFile = taosOpenFile(pCfg->path, TD_FILE_READ | TD_FILE_WRITE);
+  assert(pCfg->pFile != NULL);
+
+  taosLSeekFile(pCfg->pFile, 0, SEEK_SET);
+
+  char buf[1024];
+  int  len = taosReadFile(pCfg->pFile, buf, sizeof(buf));
+  assert(len > 0);
+
+  int32_t ret = syncCfgFromStr(buf, &(pCfg->cfg));
+  assert(ret == 0);
+
+  return pCfg;
+}
+
+int32_t raftCfgClose(SRaftCfg *pRaftCfg) {
+  int64_t ret = taosCloseFile(&(pRaftCfg->pFile));
+  assert(ret == 0);
+  taosMemoryFree(pRaftCfg);
+  return 0;
+}
+
+int32_t raftCfgPersist(SRaftCfg *pRaftCfg) {
+  assert(pRaftCfg != NULL);
+
+  char *s = syncCfg2Str(&(pRaftCfg->cfg));
+  taosLSeekFile(pRaftCfg->pFile, 0, SEEK_SET);
+  int64_t ret = taosWriteFile(pRaftCfg->pFile, s, strlen(s) + 1);
+  assert(ret == strlen(s) + 1);
+  taosMemoryFree(s);
+
+  taosFsyncFile(pRaftCfg->pFile);
+  return 0;
+}
+
+cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) {
+  char   u64buf[128];
+  cJSON *pRoot = cJSON_CreateObject();
+
+  if (pSyncCfg != NULL) {
+    cJSON_AddNumberToObject(pRoot, "replicaNum", pSyncCfg->replicaNum);
+    cJSON_AddNumberToObject(pRoot, "myIndex", pSyncCfg->myIndex);
+
+    cJSON *pNodeInfoArr = cJSON_CreateArray();
+    cJSON_AddItemToObject(pRoot, "nodeInfo", pNodeInfoArr);
+    for (int i = 0; i < pSyncCfg->replicaNum; ++i) {
+      cJSON *pNodeInfo = cJSON_CreateObject();
+      cJSON_AddNumberToObject(pNodeInfo, "nodePort", ((pSyncCfg->nodeInfo)[i]).nodePort);
+      cJSON_AddStringToObject(pNodeInfo, "nodeFqdn", ((pSyncCfg->nodeInfo)[i]).nodeFqdn);
+      cJSON_AddItemToArray(pNodeInfoArr, pNodeInfo);
+    }
+  }
+
+  cJSON *pJson = cJSON_CreateObject();
+  cJSON_AddItemToObject(pJson, "SSyncCfg", pRoot);
+  return pJson;
+}
+
+char *syncCfg2Str(SSyncCfg *pSyncCfg) {
+  cJSON *pJson = syncCfg2Json(pSyncCfg);
+  char * serialized = cJSON_Print(pJson);
+  cJSON_Delete(pJson);
+  return serialized;
+}
+
+int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) {
+  memset(pSyncCfg, 0, sizeof(SSyncCfg));
+  cJSON *pJson = cJSON_GetObjectItem(pRoot, "SSyncCfg");
+
+  cJSON *pReplicaNum = cJSON_GetObjectItem(pJson, "replicaNum");
+  assert(cJSON_IsNumber(pReplicaNum));
+  pSyncCfg->replicaNum = cJSON_GetNumberValue(pReplicaNum);
+
+  cJSON *pMyIndex = cJSON_GetObjectItem(pJson, "myIndex");
+  assert(cJSON_IsNumber(pMyIndex));
+  pSyncCfg->myIndex = cJSON_GetNumberValue(pMyIndex);
+
+  cJSON *pNodeInfoArr = cJSON_GetObjectItem(pJson, "nodeInfo");
+  int    arraySize = cJSON_GetArraySize(pNodeInfoArr);
+  assert(arraySize == pSyncCfg->replicaNum);
+
+  for (int i = 0; i < arraySize; ++i) {
+    cJSON *pNodeInfo = cJSON_GetArrayItem(pNodeInfoArr, i);
+    assert(pNodeInfo != NULL);
+
+    cJSON *pNodePort = cJSON_GetObjectItem(pNodeInfo, "nodePort");
+    assert(cJSON_IsNumber(pNodePort));
+    ((pSyncCfg->nodeInfo)[i]).nodePort = cJSON_GetNumberValue(pNodePort);
+
+    cJSON *pNodeFqdn = cJSON_GetObjectItem(pNodeInfo, "nodeFqdn");
+    assert(cJSON_IsString(pNodeFqdn));
+    snprintf(((pSyncCfg->nodeInfo)[i]).nodeFqdn, sizeof(((pSyncCfg->nodeInfo)[i]).nodeFqdn), "%s",
+             pNodeFqdn->valuestring);
+  }
+
+  return 0;
+}
+
+int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg) {
+  cJSON *pRoot = cJSON_Parse(s);
+  assert(pRoot != NULL);
+
+  int32_t ret = syncCfgFromJson(pRoot, pSyncCfg);
+  assert(ret == 0);
+
+  cJSON_Delete(pRoot);
+  return 0;
+}
+
+cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) {
+  cJSON *pJson = syncCfg2Json(&(pRaftCfg->cfg));
+  return pJson;
+}
+
+char *raftCfg2Str(SRaftCfg *pRaftCfg) {
+  char *s = syncCfg2Str(&(pRaftCfg->cfg));
+  return s;
+}
+
+int32_t syncCfgCreateFile(SSyncCfg *pCfg, const char *path) {
+  assert(pCfg != NULL);
+
+  TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE);
+  assert(pFile != NULL);
+
+  char *  s = syncCfg2Str(pCfg);
+  int64_t ret = taosWriteFile(pFile, s, strlen(s) + 1);
+  assert(ret == strlen(s) + 1);
+
+  taosMemoryFree(s);
+  taosCloseFile(&pFile);
+  return 0;
+}
+
+// for debug ----------------------
+void syncCfgPrint(SSyncCfg *pCfg) {
+  char *serialized = syncCfg2Str(pCfg);
+  printf("syncCfgPrint | len:%lu | %s \n", strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncCfgPrint2(char *s, SSyncCfg *pCfg) {
+  char *serialized = syncCfg2Str(pCfg);
+  printf("syncCfgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void syncCfgLog(SSyncCfg *pCfg) {
+  char *serialized = syncCfg2Str(pCfg);
+  sTrace("syncCfgLog | len:%lu | %s", strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void syncCfgLog2(char *s, SSyncCfg *pCfg) {
+  char *serialized = syncCfg2Str(pCfg);
+  sTrace("syncCfgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+  taosMemoryFree(serialized);
+}
+
+void raftCfgPrint(SRaftCfg *pCfg) {
+  char *serialized = raftCfg2Str(pCfg);
+  printf("raftCfgPrint | len:%lu | %s \n", strlen(serialized), serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void raftCfgPrint2(char *s, SRaftCfg *pCfg) {
+  char *serialized = raftCfg2Str(pCfg);
+  printf("raftCfgPrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
+  fflush(NULL);
+  taosMemoryFree(serialized);
+}
+
+void raftCfgLog(SRaftCfg *pCfg) {
+  char *serialized = raftCfg2Str(pCfg);
+  sTrace("raftCfgLog | len:%lu | %s", strlen(serialized), serialized);
+  taosMemoryFree(serialized);
+}
+
+void raftCfgLog2(char *s, SRaftCfg *pCfg) {
+  char *serialized = raftCfg2Str(pCfg);
+  sTrace("raftCfgLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
+  taosMemoryFree(serialized);
+}

source/libs/sync/src/syncRaftEntry.c

@@ -28,13 +28,13 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) {
 
 // step 4. SyncClientRequest => SSyncRaftEntry, add term, index
 SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) {
-  SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index, SYNC_RAFT_ENTRY_DATA);
+  SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index);
   assert(pEntry != NULL);
 
   return pEntry;
 }
 
-SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index, EntryType entryType) {
+SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) {
   SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen);
   assert(pEntry != NULL);
 
@@ -44,19 +44,37 @@ SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncInde
   pEntry->isWeak = pMsg->isWeak;
   pEntry->term = term;
   pEntry->index = index;
-  pEntry->entryType = entryType;
   pEntry->dataLen = pMsg->dataLen;
   memcpy(pEntry->data, pMsg->data, pMsg->dataLen);
 
   return pEntry;
 }
 
-SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index) {
-  SSyncRaftEntry* pEntry = syncEntryBuild(0);
+SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) {
+  // init rpcMsg
+  SMsgHead head;
+  head.vgId = vgId;
+  head.contLen = sizeof(SMsgHead);
+  SRpcMsg rpcMsg;
+  memset(&rpcMsg, 0, sizeof(SRpcMsg));
+  rpcMsg.contLen = head.contLen;
+  rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+  rpcMsg.msgType = TDMT_VND_SYNC_NOOP;
+  memcpy(rpcMsg.pCont, &head, sizeof(head));
+
+  SSyncRaftEntry* pEntry = syncEntryBuild(rpcMsg.contLen);
   assert(pEntry != NULL);
+
+  pEntry->msgType = TDMT_VND_SYNC_CLIENT_REQUEST;
+  pEntry->originalRpcType = TDMT_VND_SYNC_NOOP;
+  pEntry->seqNum = 0;
+  pEntry->isWeak = 0;
   pEntry->term = term;
   pEntry->index = index;
-  pEntry->entryType = SYNC_RAFT_ENTRY_NOOP;
+
+  assert(pEntry->dataLen == rpcMsg.contLen);
+  memcpy(pEntry->data, rpcMsg.pCont, rpcMsg.contLen);
+  rpcFreeCont(rpcMsg.pCont);
 
   return pEntry;
 }
@@ -103,7 +121,6 @@ cJSON* syncEntry2Json(const SSyncRaftEntry* pEntry) {
     cJSON_AddStringToObject(pRoot, "term", u64buf);
     snprintf(u64buf, sizeof(u64buf), "%lu", pEntry->index);
     cJSON_AddStringToObject(pRoot, "index", u64buf);
-    cJSON_AddNumberToObject(pRoot, "entryType", pEntry->entryType);
     cJSON_AddNumberToObject(pRoot, "dataLen", pEntry->dataLen);
 
     char* s;

source/libs/sync/src/syncRaftLog.c

@@ -50,38 +50,46 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) {
 
   SyncIndex lastIndex = logStoreLastIndex(pLogStore);
   assert(pEntry->index == lastIndex + 1);
-  uint32_t len;
-  char*    serialized = syncEntrySerialize(pEntry, &len);
-  assert(serialized != NULL);
 
-  int code = 0;
-  /*
-    code = walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len);
-    assert(code == 0);
-  */
-  assert(walWrite(pWal, pEntry->index, pEntry->entryType, serialized, len) == 0);
+  int          code = 0;
+  SSyncLogMeta syncMeta;
+  syncMeta.isWeek = pEntry->isWeak;
+  syncMeta.seqNum = pEntry->seqNum;
+  syncMeta.term = pEntry->term;
+  code = walWriteWithSyncInfo(pWal, pEntry->index, pEntry->originalRpcType, syncMeta, pEntry->data, pEntry->dataLen);
+  assert(code == 0);
 
   walFsync(pWal, true);
-  taosMemoryFree(serialized);
   return code;
 }
 
 SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) {
   SSyncLogStoreData* pData = pLogStore->data;
   SWal*              pWal = pData->pWal;
-  SSyncRaftEntry*    pEntry = NULL;
 
   if (index >= SYNC_INDEX_BEGIN && index <= logStoreLastIndex(pLogStore)) {
     SWalReadHandle* pWalHandle = walOpenReadHandle(pWal);
     assert(walReadWithHandle(pWalHandle, index) == 0);
-    pEntry = syncEntryDeserialize(pWalHandle->pHead->head.body, pWalHandle->pHead->head.len);
+
+    SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.len);
     assert(pEntry != NULL);
 
+    pEntry->msgType = TDMT_VND_SYNC_CLIENT_REQUEST;
+    pEntry->originalRpcType = pWalHandle->pHead->head.msgType;
+    pEntry->seqNum = pWalHandle->pHead->head.syncMeta.seqNum;
+    pEntry->isWeak = pWalHandle->pHead->head.syncMeta.isWeek;
+    pEntry->term = pWalHandle->pHead->head.syncMeta.term;
+    pEntry->index = index;
+    assert(pEntry->dataLen == pWalHandle->pHead->head.len);
+    memcpy(pEntry->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.len);
+
     // need to hold, do not new every time!!
     walCloseReadHandle(pWalHandle);
-  }
+    return pEntry;
 
-  return pEntry;
+  } else {
+    return NULL;
+  }
 }
 
 int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) {
@@ -207,20 +215,20 @@ void logStorePrint(SSyncLogStore* pLogStore) {
 
 void logStorePrint2(char* s, SSyncLogStore* pLogStore) {
   char* serialized = logStore2Str(pLogStore);
-  printf("logStorePrint | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
+  printf("logStorePrint2 | len:%lu | %s | %s \n", strlen(serialized), s, serialized);
   fflush(NULL);
   taosMemoryFree(serialized);
 }
 
 void logStoreLog(SSyncLogStore* pLogStore) {
   char* serialized = logStore2Str(pLogStore);
-  sTrace("logStorePrint | len:%lu | %s", strlen(serialized), serialized);
+  sTraceLong("logStoreLog | len:%lu | %s", strlen(serialized), serialized);
   taosMemoryFree(serialized);
 }
 
 void logStoreLog2(char* s, SSyncLogStore* pLogStore) {
   char* serialized = logStore2Str(pLogStore);
-  sTrace("logStorePrint | len:%lu | %s | %s", strlen(serialized), s, serialized);
+  sTraceLong("logStoreLog2 | len:%lu | %s | %s", strlen(serialized), s, serialized);
   taosMemoryFree(serialized);
 }
 

source/libs/sync/src/syncReplication.c

@@ -77,7 +77,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {
     SyncAppendEntries* pMsg = NULL;
     SSyncRaftEntry*    pEntry = logStoreGetEntry(pSyncNode->pLogStore, nextIndex);
     if (pEntry != NULL) {
-      pMsg = syncAppendEntriesBuild(pEntry->bytes);
+      pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId);
       assert(pMsg != NULL);
 
       // add pEntry into msg
@@ -91,7 +91,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) {
 
     } else {
       // maybe overflow, send empty record
-      pMsg = syncAppendEntriesBuild(0);
+      pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId);
       assert(pMsg != NULL);
     }
 

source/libs/sync/src/syncRequestVote.c

@@ -64,7 +64,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) {
     raftStoreVote(ths->pRaftStore, &(pMsg->srcId));
   }
 
-  SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild();
+  SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId);
   pReply->srcId = ths->myRaftId;
   pReply->destId = pMsg->srcId;
   pReply->term = ths->pRaftStore->currentTerm;

source/libs/sync/src/syncRequestVoteReply.c

@@ -49,11 +49,20 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg)
     return ret;
   }
 
-  assert(!(pMsg->term > ths->pRaftStore->currentTerm));
-  // no need this code, because if I receive reply.term, then I must have sent for that term.
-  //  if (pMsg->term > ths->pRaftStore->currentTerm) {
-  //    syncNodeUpdateTerm(ths, pMsg->term);
-  //  }
+  // assert(!(pMsg->term > ths->pRaftStore->currentTerm));
+  //  no need this code, because if I receive reply.term, then I must have sent for that term.
+  //   if (pMsg->term > ths->pRaftStore->currentTerm) {
+  //     syncNodeUpdateTerm(ths, pMsg->term);
+  //   }
+
+  if (pMsg->term > ths->pRaftStore->currentTerm) {
+    char logBuf[128];
+    snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplyCb error term, receive:%lu current:%lu", pMsg->term,
+             ths->pRaftStore->currentTerm);
+    syncNodePrint2(logBuf, ths);
+    sError("%s", logBuf);
+    return ret;
+  }
 
   assert(pMsg->term == ths->pRaftStore->currentTerm);
 

source/libs/sync/src/syncRespMgr.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "syncRespMgr.h"
+
+SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) {
+  SSyncRespMgr *pObj = (SSyncRespMgr *)taosMemoryMalloc(sizeof(SSyncRespMgr));
+  memset(pObj, 0, sizeof(SSyncRespMgr));
+
+  pObj->pRespHash =
+      taosHashInit(sizeof(uint64_t), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
+  assert(pObj->pRespHash != NULL);
+  pObj->ttl = ttl;
+  pObj->data = data;
+  pObj->seqNum = 0;
+  taosThreadMutexInit(&(pObj->mutex), NULL);
+
+  return pObj;
+}
+
+void syncRespMgrDestroy(SSyncRespMgr *pObj) {
+  taosThreadMutexLock(&(pObj->mutex));
+  taosHashCleanup(pObj->pRespHash);
+  taosThreadMutexUnlock(&(pObj->mutex));
+  taosThreadMutexDestroy(&(pObj->mutex));
+  taosMemoryFree(pObj);
+}
+
+int64_t syncRespMgrAdd(SSyncRespMgr *pObj, SRespStub *pStub) {
+  taosThreadMutexLock(&(pObj->mutex));
+
+  uint64_t keyCode = ++(pObj->seqNum);
+  taosHashPut(pObj->pRespHash, &keyCode, sizeof(keyCode), pStub, sizeof(SRespStub));
+
+  taosThreadMutexUnlock(&(pObj->mutex));
+  return keyCode;
+}
+
+int32_t syncRespMgrDel(SSyncRespMgr *pObj, uint64_t index) {
+  taosThreadMutexLock(&(pObj->mutex));
+
+  taosHashRemove(pObj->pRespHash, &index, sizeof(index));
+
+  taosThreadMutexUnlock(&(pObj->mutex));
+  return 0;
+}
+
+int32_t syncRespMgrGet(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) {
+  taosThreadMutexLock(&(pObj->mutex));
+
+  void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index));
+  if (pTmp != NULL) {
+    memcpy(pStub, pTmp, sizeof(SRespStub));
+    taosThreadMutexUnlock(&(pObj->mutex));
+    return 1;  // get one object
+  }
+  taosThreadMutexUnlock(&(pObj->mutex));
+  return 0;  // get none object
+}
+
+int32_t syncRespMgrGetAndDel(SSyncRespMgr *pObj, uint64_t index, SRespStub *pStub) {
+  taosThreadMutexLock(&(pObj->mutex));
+
+  void *pTmp = taosHashGet(pObj->pRespHash, &index, sizeof(index));
+  if (pTmp != NULL) {
+    memcpy(pStub, pTmp, sizeof(SRespStub));
+    taosThreadMutexUnlock(&(pObj->mutex));
+    taosHashRemove(pObj->pRespHash, &index, sizeof(index));
+    return 1;  // get one object
+  }
+  taosThreadMutexUnlock(&(pObj->mutex));
+  return 0;  // get none object
+}
+
+void syncRespClean(SSyncRespMgr *pObj) {
+  taosThreadMutexLock(&(pObj->mutex));
+  syncRespCleanByTTL(pObj, pObj->ttl);
+  taosThreadMutexUnlock(&(pObj->mutex));
+}
+
+void syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl) {}
\ No newline at end of file

source/libs/sync/src/syncUtil.c

@@ -16,6 +16,8 @@
 #include "syncUtil.h"
 #include "syncEnv.h"
 
+void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port);
+
 // ---- encode / decode
 uint64_t syncUtilAddr2U64(const char* host, uint16_t port) {
   uint64_t u64;
@@ -197,3 +199,17 @@ SyncIndex syncUtilMaxIndex(SyncIndex a, SyncIndex b) {
   SyncIndex r = a > b ? a : b;
   return r;
 }
+
+void syncUtilMsgHtoN(void* msg) {
+  // htonl
+  SMsgHead* pHead = msg;
+  pHead->contLen = htonl(pHead->contLen);
+  pHead->vgId = htonl(pHead->vgId);
+}
+
+void syncUtilMsgNtoH(void* msg) {
+  // ntohl
+  SMsgHead* pHead = msg;
+  pHead->contLen = ntohl(pHead->contLen);
+  pHead->vgId = ntohl(pHead->vgId);
+}
\ No newline at end of file

source/libs/sync/test/CMakeLists.txt

@@ -28,16 +28,15 @@ add_executable(syncRpcMsgTest "")
 add_executable(syncPingTimerTest2 "")
 add_executable(syncPingSelfTest "")
 add_executable(syncElectTest "")
-add_executable(syncElectTest2 "")
-add_executable(syncElectTest3 "")
 add_executable(syncEncodeTest "")
 add_executable(syncWriteTest "")
 add_executable(syncReplicateTest "")
-add_executable(syncReplicateTest2 "")
-add_executable(syncReplicateTest3 "")
-add_executable(syncReplicateLoadTest "")
 add_executable(syncRefTest "")
 add_executable(syncLogStoreCheck "")
+add_executable(syncRaftCfgTest "")
+add_executable(syncRespMgrTest "")
+add_executable(syncSnapshotTest "")
+add_executable(syncApplyMsgTest "")
 
 
 target_sources(syncTest
@@ -160,14 +159,6 @@ target_sources(syncElectTest
     PRIVATE
     "syncElectTest.cpp"
 )
-target_sources(syncElectTest2
-    PRIVATE
-    "syncElectTest2.cpp"
-)
-target_sources(syncElectTest3
-    PRIVATE
-    "syncElectTest3.cpp"
-)
 target_sources(syncEncodeTest
     PRIVATE
     "syncEncodeTest.cpp"
@@ -180,226 +171,225 @@ target_sources(syncReplicateTest
     PRIVATE
     "syncReplicateTest.cpp"
 )
-target_sources(syncReplicateTest2
+target_sources(syncRefTest
     PRIVATE
-    "syncReplicateTest2.cpp"
+    "syncRefTest.cpp"
 )
-target_sources(syncReplicateTest3
+target_sources(syncLogStoreCheck
     PRIVATE
-    "syncReplicateTest3.cpp"
+    "syncLogStoreCheck.cpp"
 )
-target_sources(syncReplicateLoadTest
+target_sources(syncRaftCfgTest
     PRIVATE
-    "syncReplicateLoadTest.cpp"
+    "syncRaftCfgTest.cpp"
 )
-target_sources(syncRefTest
+target_sources(syncRespMgrTest
     PRIVATE
-    "syncRefTest.cpp"
+    "syncRespMgrTest.cpp"
 )
-target_sources(syncLogStoreCheck
+target_sources(syncSnapshotTest
     PRIVATE
-    "syncLogStoreCheck.cpp"
+    "syncSnapshotTest.cpp"
+)
+target_sources(syncApplyMsgTest
+    PRIVATE
+    "syncApplyMsgTest.cpp"
 )
 
 
 target_include_directories(syncTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncEnvTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncPingTimerTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIOTickQTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIOTickPingTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIOSendMsgTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIOClientTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIOServerTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncRaftStoreTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncEnqTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIndexTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncInitTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncUtilTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncVotesGrantedTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncVotesRespondTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncIndexMgrTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncLogStoreTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncEntryTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncRequestVoteTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncRequestVoteReplyTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncAppendEntriesTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncAppendEntriesReplyTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncClientRequestTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncTimeoutTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncPingTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncPingReplyTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncRpcMsgTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncPingTimerTest2
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncPingSelfTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncElectTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
-)
-target_include_directories(syncElectTest2
-    PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
-)
-target_include_directories(syncElectTest3
-    PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncEncodeTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncWriteTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 target_include_directories(syncReplicateTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
-target_include_directories(syncReplicateTest2
+target_include_directories(syncRefTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
-target_include_directories(syncReplicateTest3
+target_include_directories(syncLogStoreCheck
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
-target_include_directories(syncReplicateLoadTest
+target_include_directories(syncRaftCfgTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
-target_include_directories(syncRefTest
+target_include_directories(syncRespMgrTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
-target_include_directories(syncLogStoreCheck
+target_include_directories(syncSnapshotTest
+    PUBLIC
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncApplyMsgTest
     PUBLIC
-    "${TD_SOURCE_DIR}/include/libs/sync"
+    "${CMAKE_SOURCE_DIR}/include/libs/sync"
     "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
 )
 
@@ -524,14 +514,6 @@ target_link_libraries(syncElectTest
     sync
     gtest_main
 )
-target_link_libraries(syncElectTest2
-    sync
-    gtest_main
-)
-target_link_libraries(syncElectTest3
-    sync
-    gtest_main
-)
 target_link_libraries(syncEncodeTest
     sync
     gtest_main
@@ -544,23 +526,27 @@ target_link_libraries(syncReplicateTest
     sync
     gtest_main
 )
-target_link_libraries(syncReplicateTest2
+target_link_libraries(syncRefTest
+    sync
+    gtest_main
+)
+target_link_libraries(syncLogStoreCheck
     sync
     gtest_main
 )
-target_link_libraries(syncReplicateTest3
+target_link_libraries(syncRaftCfgTest
     sync
     gtest_main
 )
-target_link_libraries(syncReplicateLoadTest
+target_link_libraries(syncRespMgrTest
     sync
     gtest_main
 )
-target_link_libraries(syncRefTest
+target_link_libraries(syncSnapshotTest
     sync
     gtest_main
 )
-target_link_libraries(syncLogStoreCheck
+target_link_libraries(syncApplyMsgTest
     sync
     gtest_main
 )

source/libs/sync/test/syncAppendEntriesReplyTest.cpp

@@ -15,7 +15,7 @@ void logTest() {
 }
 
 SyncAppendEntriesReply *createMsg() {
-  SyncAppendEntriesReply *pMsg = syncAppendEntriesReplyBuild();
+  SyncAppendEntriesReply *pMsg = syncAppendEntriesReplyBuild(1000);
   pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
   pMsg->srcId.vgId = 100;
   pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
@@ -27,7 +27,7 @@ SyncAppendEntriesReply *createMsg() {
 
 void test1() {
   SyncAppendEntriesReply *pMsg = createMsg();
-  syncAppendEntriesReplyPrint2((char *)"test1:", pMsg);
+  syncAppendEntriesReplyLog2((char *)"test1:", pMsg);
   syncAppendEntriesReplyDestroy(pMsg);
 }
 
@@ -36,10 +36,10 @@ void test2() {
   uint32_t                len = pMsg->bytes;
   char *                  serialized = (char *)taosMemoryMalloc(len);
   syncAppendEntriesReplySerialize(pMsg, serialized, len);
-  SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild();
+  SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild(1000);
   syncAppendEntriesReplyDeserialize(serialized, len, pMsg2);
-  syncAppendEntriesReplyPrint2((char *)"test2: syncAppendEntriesReplySerialize -> syncAppendEntriesReplyDeserialize ",
-                               pMsg2);
+  syncAppendEntriesReplyLog2((char *)"test2: syncAppendEntriesReplySerialize -> syncAppendEntriesReplyDeserialize ",
+                             pMsg2);
 
   taosMemoryFree(serialized);
   syncAppendEntriesReplyDestroy(pMsg);
@@ -51,8 +51,8 @@ void test3() {
   uint32_t                len;
   char *                  serialized = syncAppendEntriesReplySerialize2(pMsg, &len);
   SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyDeserialize2(serialized, len);
-  syncAppendEntriesReplyPrint2((char *)"test3: syncAppendEntriesReplySerialize3 -> syncAppendEntriesReplyDeserialize2 ",
-                               pMsg2);
+  syncAppendEntriesReplyLog2((char *)"test3: syncAppendEntriesReplySerialize3 -> syncAppendEntriesReplyDeserialize2 ",
+                             pMsg2);
 
   taosMemoryFree(serialized);
   syncAppendEntriesReplyDestroy(pMsg);
@@ -63,11 +63,12 @@ void test4() {
   SyncAppendEntriesReply *pMsg = createMsg();
   SRpcMsg                 rpcMsg;
   syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg);
-  SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild();
+  SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyBuild(1000);
   syncAppendEntriesReplyFromRpcMsg(&rpcMsg, pMsg2);
-  syncAppendEntriesReplyPrint2((char *)"test4: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg ",
-                               pMsg2);
+  syncAppendEntriesReplyLog2((char *)"test4: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg ",
+                             pMsg2);
 
+  rpcFreeCont(rpcMsg.pCont);
   syncAppendEntriesReplyDestroy(pMsg);
   syncAppendEntriesReplyDestroy(pMsg2);
 }
@@ -77,17 +78,17 @@ void test5() {
   SRpcMsg                 rpcMsg;
   syncAppendEntriesReply2RpcMsg(pMsg, &rpcMsg);
   SyncAppendEntriesReply *pMsg2 = syncAppendEntriesReplyFromRpcMsg2(&rpcMsg);
-  syncAppendEntriesReplyPrint2((char *)"test5: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg2 ",
-                               pMsg2);
+  syncAppendEntriesReplyLog2((char *)"test5: syncAppendEntriesReply2RpcMsg -> syncAppendEntriesReplyFromRpcMsg2 ",
+                             pMsg2);
 
+  rpcFreeCont(rpcMsg.pCont);
   syncAppendEntriesReplyDestroy(pMsg);
   syncAppendEntriesReplyDestroy(pMsg2);
 }
 
 int main() {
-  // taosInitLog((char *)"syncTest.log", 100000, 10);
   tsAsyncLog = 0;
-  sDebugFlag = 143 + 64;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
   logTest();
 
   test1();

source/libs/sync/test/syncAppendEntriesTest.cpp

@@ -15,7 +15,7 @@ void logTest() {
 }
 
 SyncAppendEntries *createMsg() {
-  SyncAppendEntries *pMsg = syncAppendEntriesBuild(20);
+  SyncAppendEntries *pMsg = syncAppendEntriesBuild(20, 1000);
   pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
   pMsg->srcId.vgId = 100;
   pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
@@ -29,7 +29,7 @@ SyncAppendEntries *createMsg() {
 
 void test1() {
   SyncAppendEntries *pMsg = createMsg();
-  syncAppendEntriesPrint2((char *)"test1:", pMsg);
+  syncAppendEntriesLog2((char *)"test1:", pMsg);
   syncAppendEntriesDestroy(pMsg);
 }
 
@@ -38,9 +38,9 @@ void test2() {
   uint32_t           len = pMsg->bytes;
   char *             serialized = (char *)taosMemoryMalloc(len);
   syncAppendEntriesSerialize(pMsg, serialized, len);
-  SyncAppendEntries *pMsg2 = syncAppendEntriesBuild(pMsg->dataLen);
+  SyncAppendEntries *pMsg2 = syncAppendEntriesBuild(pMsg->dataLen, 1000);
   syncAppendEntriesDeserialize(serialized, len, pMsg2);
-  syncAppendEntriesPrint2((char *)"test2: syncAppendEntriesSerialize -> syncAppendEntriesDeserialize ", pMsg2);
+  syncAppendEntriesLog2((char *)"test2: syncAppendEntriesSerialize -> syncAppendEntriesDeserialize ", pMsg2);
 
   taosMemoryFree(serialized);
   syncAppendEntriesDestroy(pMsg);
@@ -52,7 +52,7 @@ void test3() {
   uint32_t           len;
   char *             serialized = syncAppendEntriesSerialize2(pMsg, &len);
   SyncAppendEntries *pMsg2 = syncAppendEntriesDeserialize2(serialized, len);
-  syncAppendEntriesPrint2((char *)"test3: syncAppendEntriesSerialize3 -> syncAppendEntriesDeserialize2 ", pMsg2);
+  syncAppendEntriesLog2((char *)"test3: syncAppendEntriesSerialize3 -> syncAppendEntriesDeserialize2 ", pMsg2);
 
   taosMemoryFree(serialized);
   syncAppendEntriesDestroy(pMsg);
@@ -65,8 +65,9 @@ void test4() {
   syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
   SyncAppendEntries *pMsg2 = (SyncAppendEntries *)taosMemoryMalloc(rpcMsg.contLen);
   syncAppendEntriesFromRpcMsg(&rpcMsg, pMsg2);
-  syncAppendEntriesPrint2((char *)"test4: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg ", pMsg2);
+  syncAppendEntriesLog2((char *)"test4: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg ", pMsg2);
 
+  rpcFreeCont(rpcMsg.pCont);
   syncAppendEntriesDestroy(pMsg);
   syncAppendEntriesDestroy(pMsg2);
 }
@@ -76,16 +77,16 @@ void test5() {
   SRpcMsg            rpcMsg;
   syncAppendEntries2RpcMsg(pMsg, &rpcMsg);
   SyncAppendEntries *pMsg2 = syncAppendEntriesFromRpcMsg2(&rpcMsg);
-  syncAppendEntriesPrint2((char *)"test5: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg2 ", pMsg2);
+  syncAppendEntriesLog2((char *)"test5: syncAppendEntries2RpcMsg -> syncAppendEntriesFromRpcMsg2 ", pMsg2);
 
+  rpcFreeCont(rpcMsg.pCont);
   syncAppendEntriesDestroy(pMsg);
   syncAppendEntriesDestroy(pMsg2);
 }
 
 int main() {
-  // taosInitLog((char *)"syncTest.log", 100000, 10);
   tsAsyncLog = 0;
-  sDebugFlag = 143 + 64;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
   logTest();
 
   test1();

source/libs/sync/test/syncApplyMsgTest.cpp

+#include <gtest/gtest.h>
+#include <stdio.h>
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
+
+void logTest() {
+  sTrace("--- sync log test: trace");
+  sDebug("--- sync log test: debug");
+  sInfo("--- sync log test: info");
+  sWarn("--- sync log test: warn");
+  sError("--- sync log test: error");
+  sFatal("--- sync log test: fatal");
+}
+
+SyncApplyMsg *createMsg() {
+  SRpcMsg rpcMsg;
+  memset(&rpcMsg, 0, sizeof(rpcMsg));
+  rpcMsg.msgType = 12345;
+  rpcMsg.contLen = 20;
+  rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+  strcpy((char *)rpcMsg.pCont, "hello rpc");
+
+  SFsmCbMeta meta;
+  meta.code = 11;
+  meta.index = 22;
+  meta.isWeak = 1;
+  meta.seqNum = 33;
+  meta.state = TAOS_SYNC_STATE_LEADER;
+
+  SyncApplyMsg *pMsg = syncApplyMsgBuild2(&rpcMsg, 123, &meta);
+  rpcFreeCont(rpcMsg.pCont);
+  return pMsg;
+}
+
+void test1() {
+  SyncApplyMsg *pMsg = createMsg();
+  syncApplyMsgLog2((char *)"test1:", pMsg);
+  syncApplyMsgDestroy(pMsg);
+}
+
+void test2() {
+  SyncApplyMsg *pMsg = createMsg();
+  uint32_t      len = pMsg->bytes;
+  char *        serialized = (char *)taosMemoryMalloc(len);
+  syncApplyMsgSerialize(pMsg, serialized, len);
+  SyncApplyMsg *pMsg2 = syncApplyMsgBuild(pMsg->dataLen);
+  syncApplyMsgDeserialize(serialized, len, pMsg2);
+  syncApplyMsgLog2((char *)"test2: syncApplyMsgSerialize -> syncApplyMsgDeserialize ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncApplyMsgDestroy(pMsg);
+  syncApplyMsgDestroy(pMsg2);
+}
+
+void test3() {
+  SyncApplyMsg *pMsg = createMsg();
+  uint32_t      len;
+  char *        serialized = syncApplyMsgSerialize2(pMsg, &len);
+  SyncApplyMsg *pMsg2 = syncApplyMsgDeserialize2(serialized, len);
+  syncApplyMsgLog2((char *)"test3: syncApplyMsgSerialize2 -> syncApplyMsgDeserialize2 ", pMsg2);
+
+  taosMemoryFree(serialized);
+  syncApplyMsgDestroy(pMsg);
+  syncApplyMsgDestroy(pMsg2);
+}
+
+void test4() {
+  SyncApplyMsg *pMsg = createMsg();
+  SRpcMsg       rpcMsg;
+  syncApplyMsg2RpcMsg(pMsg, &rpcMsg);
+  SyncApplyMsg *pMsg2 = (SyncApplyMsg *)taosMemoryMalloc(rpcMsg.contLen);
+  syncApplyMsgFromRpcMsg(&rpcMsg, pMsg2);
+  syncApplyMsgLog2((char *)"test4: syncApplyMsg2RpcMsg -> syncApplyMsgFromRpcMsg ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncApplyMsgDestroy(pMsg);
+  syncApplyMsgDestroy(pMsg2);
+}
+
+void test5() {
+  SyncApplyMsg *pMsg = createMsg();
+  SRpcMsg       rpcMsg;
+  syncApplyMsg2RpcMsg(pMsg, &rpcMsg);
+  SyncApplyMsg *pMsg2 = syncApplyMsgFromRpcMsg2(&rpcMsg);
+  syncApplyMsgLog2((char *)"test5: syncClientRequest2RpcMsg -> syncApplyMsgFromRpcMsg2 ", pMsg2);
+
+  rpcFreeCont(rpcMsg.pCont);
+  syncApplyMsgDestroy(pMsg);
+  syncApplyMsgDestroy(pMsg2);
+}
+
+void test6() {
+  SyncApplyMsg *pMsg = createMsg();
+  SRpcMsg       rpcMsg;
+  syncApplyMsg2RpcMsg(pMsg, &rpcMsg);
+  SyncApplyMsg *pMsg2 = syncApplyMsgFromRpcMsg2(&rpcMsg);
+
+  SRpcMsg originalRpcMsg;
+  syncApplyMsg2OriginalRpcMsg(pMsg2, &originalRpcMsg);
+  syncRpcMsgLog2((char *)"test6", &originalRpcMsg);
+
+  rpcFreeCont(originalRpcMsg.pCont);
+  rpcFreeCont(rpcMsg.pCont);
+  syncApplyMsgDestroy(pMsg);
+  syncApplyMsgDestroy(pMsg2);
+}
+
+int main() {
+  tsAsyncLog = 0;
+  sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+  logTest();
+
+  test1();
+  test2();
+  test3();
+  test4();
+  test5();
+  test6();
+
+  return 0;
+}