eth, eth/downloader, jsre: surface state sync progress through the API

eth/api.go

@@ -152,21 +152,27 @@ func (s *PublicEthereumAPI) Hashrate() *rpc.HexNumber {
 }
 
 // Syncing returns false in case the node is currently not synching with the network. It can be up to date or has not
-// yet received the latest block headers from its pears. In case it is synchronizing an object with 3 properties is
-// returned:
+// yet received the latest block headers from its pears. In case it is synchronizing:
 // - startingBlock: block number this node started to synchronise from
-// - currentBlock: block number this node is currently importing
-// - highestBlock: block number of the highest block header this node has received from peers
+// - currentBlock:  block number this node is currently importing
+// - highestBlock:  block number of the highest block header this node has received from peers
+// - pulledStates:  number of state entries processed until now
+// - knownStates:   number of known state entries that still need to be pulled
 func (s *PublicEthereumAPI) Syncing() (interface{}, error) {
-	origin, current, height := s.e.Downloader().Progress()
-	if current < height {
-		return map[string]interface{}{
-			"startingBlock": rpc.NewHexNumber(origin),
-			"currentBlock":  rpc.NewHexNumber(current),
-			"highestBlock":  rpc.NewHexNumber(height),
-		}, nil
-	}
-	return false, nil
+	origin, current, height, pulled, known := s.e.Downloader().Progress()
+
+	// Return not syncing if the synchronisation already completed
+	if current >= height {
+		return false, nil
+	}
+	// Otherwise gather the block sync stats
+	return map[string]interface{}{
+		"startingBlock": rpc.NewHexNumber(origin),
+		"currentBlock":  rpc.NewHexNumber(current),
+		"highestBlock":  rpc.NewHexNumber(height),
+		"pulledStates":  rpc.NewHexNumber(pulled),
+		"knownStates":   rpc.NewHexNumber(known),
+	}, nil
 }
 
 // PublicMinerAPI provides an API to control the miner.

eth/downloader/api.go

@@ -36,6 +36,8 @@ type Progress struct {
 	Origin  uint64 `json:"startingBlock"`
 	Current uint64 `json:"currentBlock"`
 	Height  uint64 `json:"highestBlock"`
+	Pulled  uint64 `json:"pulledStates"`
+	Known   uint64 `json:"knownStates"`
 }
 
 // SyncingResult provides information about the current synchronisation status for this node.
@@ -44,7 +46,7 @@ type SyncingResult struct {
 	Status  Progress `json:"status"`
 }
 
-// Syncing provides information when this nodes starts synchronising with the Ethereumn network and when it's finished.
+// Syncing provides information when this nodes starts synchronising with the Ethereum network and when it's finished.
 func (s *PublicDownloaderAPI) Syncing() (rpc.Subscription, error) {
 	sub := s.d.mux.Subscribe(StartEvent{}, DoneEvent{}, FailedEvent{})
 
@@ -52,13 +54,12 @@ func (s *PublicDownloaderAPI) Syncing() (rpc.Subscription, error) {
 		switch event.(type) {
 		case StartEvent:
 			result := &SyncingResult{Syncing: true}
-			result.Status.Origin, result.Status.Current, result.Status.Height = s.d.Progress()
+			result.Status.Origin, result.Status.Current, result.Status.Height, result.Status.Pulled, result.Status.Known = s.d.Progress()
 			return result
 		case DoneEvent, FailedEvent:
 			return false
 		}
 		return nil
 	}
-
 	return rpc.NewSubscriptionWithOutputFormat(sub, output), nil
 }

eth/downloader/downloader.go

@@ -196,7 +196,15 @@ func New(stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, ha
 // Progress retrieves the synchronisation boundaries, specifically the origin
 // block where synchronisation started at (may have failed/suspended); the block
 // or header sync is currently at; and the latest known block which the sync targets.
-func (d *Downloader) Progress() (uint64, uint64, uint64) {
+//
+// In addition, during the state download phase of fast synchonisation the number
+// of processed and the total number of known states are also returned. Otherwise
+// these are zero.
+func (d *Downloader) Progress() (uint64, uint64, uint64, uint64, uint64) {
+	// Fetch the pending state count outside of the lock to prevent unforeseen deadlocks
+	pendingStates := uint64(d.queue.PendingNodeData())
+
+	// Lock the current stats and return the progress
 	d.syncStatsLock.RLock()
 	defer d.syncStatsLock.RUnlock()
 
@@ -209,7 +217,7 @@ func (d *Downloader) Progress() (uint64, uint64, uint64) {
 	case LightSync:
 		current = d.headHeader().Number.Uint64()
 	}
-	return d.syncStatsChainOrigin, current, d.syncStatsChainHeight
+	return d.syncStatsChainOrigin, current, d.syncStatsChainHeight, d.syncStatsStateDone, d.syncStatsStateDone + pendingStates
 }
 
 // Synchronising returns whether the downloader is currently retrieving blocks.
@@ -296,7 +304,7 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode
 		default:
 		}
 	}
-	// Reset and ephemeral sync statistics
+	// Reset any ephemeral sync statistics
 	d.syncStatsLock.Lock()
 	d.syncStatsStateTotal = 0
 	d.syncStatsStateDone = 0

eth/downloader/downloader_test.go

@@ -1301,7 +1301,7 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		<-progress
 	}
 	// Retrieve the sync progress and ensure they are zero (pristine sync)
-	if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
 		t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
 	}
 	// Synchronise half the blocks and check initial progress
@@ -1316,7 +1316,7 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		}
 	}()
 	<-starting
-	if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks/2+1) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks/2+1) {
 		t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks/2+1)
 	}
 	progress <- struct{}{}
@@ -1333,14 +1333,14 @@ func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		}
 	}()
 	<-starting
-	if origin, current, latest := tester.downloader.Progress(); origin != uint64(targetBlocks/2+1) || current != uint64(targetBlocks/2+1) || latest != uint64(targetBlocks) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != uint64(targetBlocks/2+1) || current != uint64(targetBlocks/2+1) || latest != uint64(targetBlocks) {
 		t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, targetBlocks/2+1, targetBlocks/2+1, targetBlocks)
 	}
 	progress <- struct{}{}
 	pending.Wait()
 
 	// Check final progress after successful sync
-	if origin, current, latest := tester.downloader.Progress(); origin != uint64(targetBlocks/2+1) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != uint64(targetBlocks/2+1) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
 		t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, targetBlocks/2+1, targetBlocks, targetBlocks)
 	}
 }
@@ -1373,7 +1373,7 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		<-progress
 	}
 	// Retrieve the sync progress and ensure they are zero (pristine sync)
-	if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
 		t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
 	}
 	// Synchronise with one of the forks and check progress
@@ -1388,7 +1388,7 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		}
 	}()
 	<-starting
-	if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(len(hashesA)-1) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(len(hashesA)-1) {
 		t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, len(hashesA)-1)
 	}
 	progress <- struct{}{}
@@ -1408,14 +1408,14 @@ func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		}
 	}()
 	<-starting
-	if origin, current, latest := tester.downloader.Progress(); origin != uint64(common) || current != uint64(len(hashesA)-1) || latest != uint64(len(hashesB)-1) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != uint64(common) || current != uint64(len(hashesA)-1) || latest != uint64(len(hashesB)-1) {
 		t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, common, len(hashesA)-1, len(hashesB)-1)
 	}
 	progress <- struct{}{}
 	pending.Wait()
 
 	// Check final progress after successful sync
-	if origin, current, latest := tester.downloader.Progress(); origin != uint64(common) || current != uint64(len(hashesB)-1) || latest != uint64(len(hashesB)-1) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != uint64(common) || current != uint64(len(hashesB)-1) || latest != uint64(len(hashesB)-1) {
 		t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, common, len(hashesB)-1, len(hashesB)-1)
 	}
 }
@@ -1448,7 +1448,7 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		<-progress
 	}
 	// Retrieve the sync progress and ensure they are zero (pristine sync)
-	if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
 		t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
 	}
 	// Attempt a full sync with a faulty peer
@@ -1468,7 +1468,7 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		}
 	}()
 	<-starting
-	if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks) {
 		t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks)
 	}
 	progress <- struct{}{}
@@ -1485,14 +1485,14 @@ func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		}
 	}()
 	<-starting
-	if origin, current, latest := tester.downloader.Progress(); origin != 0 || current > uint64(targetBlocks/2) || latest != uint64(targetBlocks) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current > uint64(targetBlocks/2) || latest != uint64(targetBlocks) {
 		t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", origin, current, latest, 0, targetBlocks/2, targetBlocks)
 	}
 	progress <- struct{}{}
 	pending.Wait()
 
 	// Check final progress after successful sync
-	if origin, current, latest := tester.downloader.Progress(); origin > uint64(targetBlocks/2) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin > uint64(targetBlocks/2) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
 		t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", origin, current, latest, targetBlocks/2, targetBlocks, targetBlocks)
 	}
 }
@@ -1524,7 +1524,7 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		<-progress
 	}
 	// Retrieve the sync progress and ensure they are zero (pristine sync)
-	if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
 		t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
 	}
 	//  Create and sync with an attacker that promises a higher chain than available
@@ -1545,7 +1545,7 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		}
 	}()
 	<-starting
-	if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks+3) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks+3) {
 		t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks+3)
 	}
 	progress <- struct{}{}
@@ -1562,14 +1562,14 @@ func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
 		}
 	}()
 	<-starting
-	if origin, current, latest := tester.downloader.Progress(); origin != 0 || current > uint64(targetBlocks) || latest != uint64(targetBlocks) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin != 0 || current > uint64(targetBlocks) || latest != uint64(targetBlocks) {
 		t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", origin, current, latest, 0, targetBlocks, targetBlocks)
 	}
 	progress <- struct{}{}
 	pending.Wait()
 
 	// Check final progress after successful sync
-	if origin, current, latest := tester.downloader.Progress(); origin > uint64(targetBlocks) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
+	if origin, current, latest, _, _ := tester.downloader.Progress(); origin > uint64(targetBlocks) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
 		t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", origin, current, latest, targetBlocks, targetBlocks, targetBlocks)
 	}
 }