[TD-10645][raft]<feature>refactor node and progress map

source/libs/sync/inc/sync_raft_node_map.h

@@ -30,7 +30,7 @@ void syncRaftClearNodeMap(SSyncRaftNodeMap* nodeMap);
 
 bool syncRaftIsInNodeMap(const SSyncRaftNodeMap* nodeMap, SyncNodeId nodeId);
 
-void syncRaftCopyNodeMap(const SSyncRaftNodeMap* nodeMap, SSyncRaftNodeMap* to);
+void syncRaftCopyNodeMap(const SSyncRaftNodeMap* from, SSyncRaftNodeMap* to);
 
 void syncRaftUnionNodeMap(const SSyncRaftNodeMap* nodeMap, SSyncRaftNodeMap* to);
 

source/libs/sync/inc/sync_raft_quorum_joint.h

@@ -59,7 +59,19 @@ static FORCE_INLINE const SSyncRaftNodeMap* syncRaftJointConfigOutgoing(const SS
 }
 
 static FORCE_INLINE void syncRaftJointConfigClearOutgoing(SSyncRaftQuorumJointConfig* config) {
-  memset(&config->outgoing, 0, sizeof(SSyncRaftNodeMap));
+  syncRaftClearNodeMap(&config->outgoing);
+}
+
+static FORCE_INLINE bool syncRaftJointConfigIsIncomingEmpty(const SSyncRaftQuorumJointConfig* config) {
+  return syncRaftNodeMapSize(&config->incoming) == 0;
+}
+
+static FORCE_INLINE bool syncRaftJointConfigIsOutgoingEmpty(const SSyncRaftQuorumJointConfig* config) {
+  return syncRaftNodeMapSize(&config->outgoing) == 0;
+}
+
+static FORCE_INLINE bool syncRaftJointConfigIsInOutgoing(const SSyncRaftQuorumJointConfig* config, SyncNodeId id) {
+  return syncRaftIsInNodeMap(&config->outgoing, id);
 }
 
 void syncRaftJointConfigIDS(const SSyncRaftQuorumJointConfig* config, SSyncRaftNodeMap* nodeMap);

source/libs/sync/src/sync_raft_config_change.c

@@ -40,40 +40,7 @@ static void makeVoter(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig*
 static void makeLearner(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
                        SSyncRaftProgressMap* progressMap, SyncNodeId id);
 static void removeNodeId(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
-                       SSyncRaftProgressMap* progressMap, SyncNodeId id);              
-// syncRaftChangerSimpleConfig carries out a series of configuration changes that (in aggregate)
-// mutates the incoming majority config Voters[0] by at most one. This method
-// will return an error if that is not the case, if the resulting quorum is
-// zero, or if the configuration is in a joint state (i.e. if there is an
-// outgoing configuration).
-int syncRaftChangerSimpleConfig(SSyncRaftChanger* changer, const SSyncConfChangeSingleArray* css,
-                            SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) {
-  int ret;
-
-  ret = checkAndCopy(changer, config, progressMap);
-  if (ret != 0) {
-    return ret;
-  }
-
-  if (hasJointConfig(config)) {
-    syncError("can't apply simple config change in joint config");
-    return -1;
-  }
-
-  ret = applyConfig(changer, config, progressMap, css);
-  if (ret != 0) {
-    return ret;
-  }
-
-  int n = symDiff(syncRaftJointConfigIncoming(&changer->tracker->config.voters),
-                  syncRaftJointConfigIncoming(&config->voters));
-  if (n > 1) {
-    syncError("more than one voter changed without entering joint config");
-    return -1;
-  }
-
-  return checkAndReturn(config, progressMap); 
-}
+                       SSyncRaftProgressMap* progressMap, SyncNodeId id);
 
 // EnterJoint verifies that the outgoing (=right) majority config of the joint
 // config is empty and initializes it with a copy of the incoming (=left)
@@ -96,12 +63,13 @@ int syncRaftChangerEnterJoint(SSyncRaftChanger* changer, bool autoLeave, const S
   if (ret != 0) {
     return ret;
   }
+
   if (hasJointConfig(config)) {
     syncError("config is already joint");
     return -1;
   }
 
-  if(syncRaftNodeMapSize(&config->voters.incoming) == 0) {
+  if(syncRaftJointConfigIsIncomingEmpty(&config->voters) == 0) {
 		// We allow adding nodes to an empty config for convenience (testing and
 		// bootstrap), but you can't enter a joint state.
     syncError("can't make a zero-voter config joint");
@@ -123,6 +91,167 @@ int syncRaftChangerEnterJoint(SSyncRaftChanger* changer, bool autoLeave, const S
   return checkAndReturn(config, progressMap);
 }
 
+// syncRaftChangerSimpleConfig carries out a series of configuration changes that (in aggregate)
+// mutates the incoming majority config Voters[0] by at most one. This method
+// will return an error if that is not the case, if the resulting quorum is
+// zero, or if the configuration is in a joint state (i.e. if there is an
+// outgoing configuration).
+int syncRaftChangerSimpleConfig(SSyncRaftChanger* changer, const SSyncConfChangeSingleArray* css,
+                            SSyncRaftProgressTrackerConfig* config, SSyncRaftProgressMap* progressMap) {
+  int ret;
+
+  ret = checkAndCopy(changer, config, progressMap);
+  if (ret != 0) {
+    return ret;
+  }
+
+  if (hasJointConfig(config)) {
+    syncError("can't apply simple config change in joint config");
+    return -1;
+  }
+
+  ret = applyConfig(changer, config, progressMap, css);
+  if (ret != 0) {
+    return ret;
+  }
+
+  int n = symDiff(syncRaftJointConfigIncoming(&changer->tracker->config.voters),
+                  syncRaftJointConfigIncoming(&config->voters));
+  if (n > 1) {
+    syncError("more than one voter changed without entering joint config");
+    return -1;
+  }
+
+  return checkAndReturn(config, progressMap); 
+}
+
+// apply a change to the configuration. By convention, changes to voters are
+// always made to the incoming majority config Voters[0]. Voters[1] is either
+// empty or preserves the outgoing majority configuration while in a joint state.
+static int applyConfig(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
+                       SSyncRaftProgressMap* progressMap, const SSyncConfChangeSingleArray* css) {
+  int i;
+
+  for (i = 0; i < css->n; ++i) {
+    const SSyncConfChangeSingle* cs = &(css->changes[i]);
+    if (cs->nodeId == SYNC_NON_NODE_ID) {
+      continue;
+    }
+
+    ESyncRaftConfChangeType type = cs->type;
+    switch (type) {
+      case SYNC_RAFT_Conf_AddNode:
+        makeVoter(changer, config, progressMap, cs->nodeId);
+        break;
+      case SYNC_RAFT_Conf_AddLearnerNode:
+        makeLearner(changer, config, progressMap, cs->nodeId);
+        break;
+      case SYNC_RAFT_Conf_RemoveNode:
+        removeNodeId(changer, config, progressMap, cs->nodeId);
+        break;
+      case SYNC_RAFT_Conf_UpdateNode:
+        break;
+    }
+  }
+
+  if (syncRaftJointConfigIsIncomingEmpty(&config->voters)) {
+    syncError("removed all voters");
+    return -1;
+  }
+
+  return 0;
+}
+
+
+// makeVoter adds or promotes the given ID to be a voter in the incoming
+// majority config.
+static void makeVoter(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
+                       SSyncRaftProgressMap* progressMap, SyncNodeId id) {
+  SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(progressMap, id);
+  if (progress == -1) {
+    initProgress(changer, config, progressMap, id, false);
+    return;
+  }
+
+  progress->isLearner = false;
+  nilAwareDelete(&config->learners, id);
+  nilAwareDelete(&config->learnersNext, id);
+  syncRaftJointConfigAddToIncoming(&config->voters, id);
+}
+
+// makeLearner makes the given ID a learner or stages it to be a learner once
+// an active joint configuration is exited.
+//
+// The former happens when the peer is not a part of the outgoing config, in
+// which case we either add a new learner or demote a voter in the incoming
+// config.
+//
+// The latter case occurs when the configuration is joint and the peer is a
+// voter in the outgoing config. In that case, we do not want to add the peer
+// as a learner because then we'd have to track a peer as a voter and learner
+// simultaneously. Instead, we add the learner to LearnersNext, so that it will
+// be added to Learners the moment the outgoing config is removed by
+// LeaveJoint().
+static void makeLearner(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
+                       SSyncRaftProgressMap* progressMap, SyncNodeId id) {
+  SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(progressMap, id);
+  if (progress == NULL) {
+    initProgress(changer, config, progressMap, id, true);
+    return;
+  }
+
+  if (progress->isLearner) {
+    return;
+  }
+  // Remove any existing voter in the incoming config...
+  removeNodeId(changer, config, progressMap, id);
+  
+  // ... but save the Progress.
+  syncRaftAddToProgressMap(progressMap, progress);
+
+	// Use LearnersNext if we can't add the learner to Learners directly, i.e.
+	// if the peer is still tracked as a voter in the outgoing config. It will
+	// be turned into a learner in LeaveJoint().
+	//
+	// Otherwise, add a regular learner right away.
+  bool inInOutgoing = syncRaftJointConfigIsInOutgoing(&config->voters, id);
+  if (inInOutgoing) {
+    nilAwareAdd(&config->learnersNext, id);
+  } else {
+    nilAwareAdd(&config->learners, id);
+    progress->isLearner = true;
+  }
+}
+
+// removeNodeId this peer as a voter or learner from the incoming config.
+static void removeNodeId(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
+                       SSyncRaftProgressMap* progressMap, SyncNodeId id) {
+  SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(progressMap, id);
+  if (progress == NULL) {
+    return;
+  }
+
+  syncRaftJointConfigRemoveFromIncoming(&config->voters, id);
+  nilAwareDelete(&config->learners, id);
+  nilAwareDelete(&config->learnersNext, id);
+
+  // If the peer is still a voter in the outgoing config, keep the Progress.
+  bool inInOutgoing = syncRaftJointConfigIsInOutgoing(&config->voters, id);
+  if (!inInOutgoing) {
+    syncRaftRemoveFromProgressMap(progressMap, id);
+  }
+}
+
+// initProgress initializes a new progress for the given node or learner.
+static void initProgress(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
+                       SSyncRaftProgressMap* progressMap, SyncNodeId id, bool isLearner) {
+  if (!isLearner) {
+    syncRaftJointConfigAddToIncoming(&config->voters, id);
+  } else {
+    nilAwareAdd(&config->learners, id);
+  }
+}
+
 // checkAndCopy copies the tracker's config and progress map (deeply enough for
 // the purposes of the Changer) and returns those copies. It returns an error
 // if checkInvariants does.
@@ -208,41 +337,7 @@ static int checkInvariants(SSyncRaftProgressTrackerConfig* config, SSyncRaftProg
 }
 
 static bool hasJointConfig(const SSyncRaftProgressTrackerConfig* config) {
-  return syncRaftNodeMapSize(&config->voters.outgoing) > 0;
-}
-
-static int applyConfig(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
-                       SSyncRaftProgressMap* progressMap, const SSyncConfChangeSingleArray* css) {
-  int i;
-
-  for (i = 0; i < css->n; ++i) {
-    const SSyncConfChangeSingle* cs = &(css->changes[i]);
-    if (cs->nodeId == SYNC_NON_NODE_ID) {
-      continue;
-    }
-
-    ESyncRaftConfChangeType type = cs->type;
-    switch (type) {
-      case SYNC_RAFT_Conf_AddNode:
-        makeVoter(changer, config, progressMap, cs->nodeId);
-        break;
-      case SYNC_RAFT_Conf_AddLearnerNode:
-        makeLearner(changer, config, progressMap, cs->nodeId);
-        break;
-      case SYNC_RAFT_Conf_RemoveNode:
-        removeNodeId(changer, config, progressMap, cs->nodeId);
-        break;
-      case SYNC_RAFT_Conf_UpdateNode:
-        break;
-    }
-  }
-
-  if (syncRaftNodeMapSize(&config->voters.incoming) == 0) {
-    syncError("removed all voters");
-    return -1;
-  }
-
-  return 0;
+  return !syncRaftJointConfigIsOutgoingEmpty(&config->voters);
 }
 
 // symdiff returns the count of the symmetric difference between the sets of
@@ -272,11 +367,6 @@ static int symDiff(const SSyncRaftNodeMap* l, const SSyncRaftNodeMap* r) {
   return n;
 }
 
-static void initProgress(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
-                       SSyncRaftProgressMap* progressMap, SyncNodeId id, bool isLearner) {
-
-}
-
 // nilAwareDelete deletes from a map, nil'ing the map itself if it is empty after.
 static void nilAwareDelete(SSyncRaftNodeMap* nodeMap, SyncNodeId id) {
   syncRaftRemoveFromNodeMap(nodeMap, id);
@@ -285,83 +375,4 @@ static void nilAwareDelete(SSyncRaftNodeMap* nodeMap, SyncNodeId id) {
 // nilAwareAdd populates a map entry, creating the map if necessary.
 static void nilAwareAdd(SSyncRaftNodeMap* nodeMap, SyncNodeId id) {
   syncRaftAddToNodeMap(nodeMap, id);
-}
-
-// makeVoter adds or promotes the given ID to be a voter in the incoming
-// majority config.
-static void makeVoter(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
-                       SSyncRaftProgressMap* progressMap, SyncNodeId id) {
-  SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(progressMap, id);
-  if (progress == -1) {
-    initProgress(changer, config, progressMap, id, false);
-    return;
-  }
-
-  progress->isLearner = false;
-  nilAwareDelete(&config->learners, id);
-  nilAwareDelete(&config->learnersNext, id);
-  syncRaftJointConfigAddToIncoming(&config->voters, id);
-}
-
-// makeLearner makes the given ID a learner or stages it to be a learner once
-// an active joint configuration is exited.
-//
-// The former happens when the peer is not a part of the outgoing config, in
-// which case we either add a new learner or demote a voter in the incoming
-// config.
-//
-// The latter case occurs when the configuration is joint and the peer is a
-// voter in the outgoing config. In that case, we do not want to add the peer
-// as a learner because then we'd have to track a peer as a voter and learner
-// simultaneously. Instead, we add the learner to LearnersNext, so that it will
-// be added to Learners the moment the outgoing config is removed by
-// LeaveJoint().
-static void makeLearner(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
-                       SSyncRaftProgressMap* progressMap, SyncNodeId id) {
-  SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(progressMap, id);
-  if (progress == NULL) {
-    initProgress(changer, config, progressMap, id, false);
-    return;
-  }
-
-  if (progress->isLearner) {
-    return;
-  }
-  // Remove any existing voter in the incoming config...
-  removeNodeId(changer, config, progressMap, id);
-  
-  // ... but save the Progress.
-  syncRaftAddToProgressMap(progressMap, progress);
-
-	// Use LearnersNext if we can't add the learner to Learners directly, i.e.
-	// if the peer is still tracked as a voter in the outgoing config. It will
-	// be turned into a learner in LeaveJoint().
-	//
-	// Otherwise, add a regular learner right away.
-  bool inOutgoing = syncRaftIsInNodeMap(&config->voters.outgoing, id);
-  if (inOutgoing) {
-    nilAwareAdd(&config->learnersNext, id);
-  } else {
-    nilAwareAdd(&config->learners, id);
-    progress->isLearner = true;
-  }
-}
-
-// removeNodeId this peer as a voter or learner from the incoming config.
-static void removeNodeId(SSyncRaftChanger* changer, SSyncRaftProgressTrackerConfig* config,
-                       SSyncRaftProgressMap* progressMap, SyncNodeId id) {
-  SSyncRaftProgress* progress = syncRaftFindProgressByNodeId(progressMap, id);
-  if (progress == NULL) {
-    return;
-  }
-
-  syncRaftJointConfigRemoveFromIncoming(&config->voters, id);
-  nilAwareDelete(&config->learners, id);
-  nilAwareDelete(&config->learnersNext, id);
-
-  // If the peer is still a voter in the outgoing config, keep the Progress.
-  bool inOutgoing = syncRaftIsInNodeMap(&config->voters.outgoing, id);
-  if (!inOutgoing) {
-    syncRaftRemoveFromProgressMap(progressMap, id);
-  }
 }
\ No newline at end of file

source/libs/sync/src/sync_raft_node_map.c

@@ -32,11 +32,11 @@ bool syncRaftIsInNodeMap(const SSyncRaftNodeMap* nodeMap, SyncNodeId nodeId) {
   return true;
 }
 
-void syncRaftCopyNodeMap(const SSyncRaftNodeMap* nodeMap, SSyncRaftNodeMap* to) {
-  SyncNodeId** ppId = (SyncNodeId**)taosHashIterate(nodeMap->nodeIdMap, NULL);
+void syncRaftCopyNodeMap(const SSyncRaftNodeMap* from, SSyncRaftNodeMap* to) {
+  SyncNodeId** ppId = (SyncNodeId**)taosHashIterate(from->nodeIdMap, NULL);
   while (ppId) {
     taosHashPut(to->nodeIdMap, ppId, sizeof(SyncNodeId*), ppId, sizeof(SyncNodeId*));
-    ppId = taosHashIterate(nodeMap->nodeIdMap, ppId);
+    ppId = taosHashIterate(from->nodeIdMap, ppId);
   }
 }