Merge pull request #676 from EOSIO/p2p-sync-catchup-674

Fix for improper coordination between active block producers and thos…

(reapplied on noon)

plugins/net_plugin/include/eos/net_plugin/net_plugin.hpp

@@ -23,7 +23,7 @@ namespace eosio {
         void plugin_shutdown();
 
         void broadcast_block(const chain::signed_block &sb);
-
+        size_t num_peers () const;
       private:
         std::unique_ptr<class net_plugin_impl> my;
    };

plugins/net_plugin/net_plugin.cpp

@@ -148,6 +148,7 @@ namespace eosio {
     void start_read_message( connection_ptr c);
 
     void close( connection_ptr c );
+    size_t count_open_sockets () const;
 
     template<typename VerifierFunc>
     void send_all (const net_message &msg, VerifierFunc verify);
@@ -467,6 +468,7 @@ namespace eosio {
   };
 
   class sync_manager {
+  public:
     uint32_t            sync_known_lib_num;
     uint32_t            sync_last_requested_num;
     uint32_t            sync_req_span;
@@ -1269,6 +1271,17 @@ namespace eosio {
       );
     }
 
+  size_t net_plugin_impl::count_open_sockets () const
+  {
+    size_t count = 0;
+    for( auto &c : connections) {
+      if (c->socket->is_open())
+        ++count;
+    }
+    return count;
+  }
+
+
     template<typename VerifierFunc>
     void net_plugin_impl::send_all( const net_message &msg, VerifierFunc verify) {
       for( auto &c : connections) {
@@ -1720,7 +1733,7 @@ namespace eosio {
 
       if( !has_chunk) {
         if (c->sync_receiving)
-          elog("got a block while syncing but sync_receiving end block == 0set #${n}",
+          elog("got a block while syncing but sync_receiving end block == 0 #${n}",
              ( "n",num));
         else
           elog("got a block while syncing but no sync_receiving set #${n}",
@@ -1728,14 +1741,15 @@ namespace eosio {
       }
       else {
         if( c->sync_receiving->last + 1 != num) {
-          elog( "expected block ${next} but got ${num}",("next",c->sync_receiving->last+1)("num",num));
+          wlog( "expected block ${next} but got ${num} ihnotinh got now",("next",c->sync_receiving->last+1)("num",num));
+          return;
         }
         c->sync_receiving->last = num;
       }
     }
     bool accepted = false;
     fc_dlog(logger, "last irreversible block = ${lib}", ("lib", cc.last_irreversible_block_num()));
-    if( !syncing || num == cc.head_block_num()+1 ){ //  || num > cc.last_irreversible_block_num()) {
+    if( !syncing || num == cc.head_block_num()+1 ) {
       try {
         chain_plug->accept_block(msg, syncing);
         accepted = true;
@@ -1763,11 +1777,11 @@ namespace eosio {
       }
     }
     else {
-      fc_dlog(logger, "forwarding the signed block");
-      if (age < fc::seconds(3) && fc::raw::pack_size(msg) < just_send_it_max && !c->syncing) {
+      if (age < fc::seconds(3) && fc::raw::pack_size(msg) < just_send_it_max && !c->syncing ) {
+        fc_dlog(logger, "forwarding the signed block");
         send_all( msg, [c, blk_id, num](connection_ptr conn) -> bool {
             bool sendit = false;
-            if ( c != conn ) {
+            if ( c != conn && !conn->syncing ) {
               auto b = conn->block_state.get<by_id>().find(blk_id);
               if (b == conn->block_state.end()) {
                 conn->block_state.insert( (block_state){blk_id,true,true,fc::time_point()});
@@ -2195,4 +2209,9 @@ namespace eosio {
       fc_dlog(my->logger, "broadcasting block #${num}",("num",sb.block_num()) );
       my->broadcast_block_impl( sb);
     }
+
+  size_t net_plugin::num_peers( ) const {
+    return my->count_open_sockets();
+  }
+
 }

plugins/producer_plugin/producer_plugin.cpp

@@ -224,6 +224,9 @@ block_production_condition::block_production_condition_enum producer_plugin_impl
    // If the next block production opportunity is in the present or future, we're synced.
    if( !_production_enabled )
    {
+     if( app().get_plugin<net_plugin>().num_peers() == 0 ) {
+       _production_enabled = true;
+     }
       if( chain.get_slot_time(1) >= now )
          _production_enabled = true;
       else

programs/launcher/main.cpp

@@ -458,7 +458,7 @@ launcher_def::generate () {
 void
 launcher_def::write_dot_file () {
   bf::ofstream df ("testnet.dot");
-  df << "digraph G\n{\nlayout=\"circo\";";
+  df << "digraph G\n{\nlayout=\"circo\";\n";
   for (auto &node : network.nodes) {
     for (const auto &p : node.second.peers) {
       string pname=network.nodes.find(p)->second.instance->dot_label();
@@ -467,6 +467,18 @@ launcher_def::write_dot_file () {
          << "\" [dir=\"forward\"];" << std::endl;
     }
   }
+  #if 0
+  // this is an experiment. I was thinking of adding a "notes" box but I
+  // can't figure out hot to force it to the lower left corner of the diagram
+  df << " { rank = sink;\n"
+     << "   Notes  [shape=none, margin=0, label=<\n"
+     << "    <TABLE BORDER=\"1\" CELLBORDER=\"0\" CELLSPACING=\"0\" CELLPADDING=\"1\">\n"
+     << "     <TR> <TD align=\"left\">Data flow between nodes is bidirectional</TD> </TR>\n"
+     << "     <TR> <TD align=\"left\">Arrows point from client to server</TD> </TR>\n"
+     << "    </TABLE>\n"
+     << "   >];"
+     << " }\n";
+#endif
   df << "}\n";
 }
 
@@ -585,7 +597,7 @@ launcher_def::write_config_file (tn_node_def &node) {
     cfg << "remote-endpoint = " << network.nodes.find(p)->second.instance->p2p_endpoint << "\n";
   }
   if (node.producers.size()) {
-    cfg << "enable-stale-production = true\n"
+    cfg << "enable-stale-production = false\n"
         << "required-participation = true\n";
     for (const auto &kp : node.keys ) {
       cfg << "private-key = [\"" << kp.public_key

tests/p2p_tests/sync/test.sh

@@ -46,7 +46,12 @@ fi
 total_nodes="${total_nodes:-`echo $pnodes`}"
 
 rm -rf tn_data_*
-programs/launcher/launcher -p $pnodes -n $total_nodes -s $topo -d $delay
+if [ "$delay" == 0 ]; then
+    programs/launcher/launcher -p $pnodes -n $total_nodes -s $topo -i now
+else
+    programs/launcher/launcher -p $pnodes -n $total_nodes -s $topo -d $delay
+fi
+
 sleep 7
 echo "start" > test.out
 port=8888