Merge netsim mig to master (#17241)

* swarm: merged stream-tests migration to develop

* swarm/network: expose simulation RandomUpNode to use in stream tests

* swarm/network: wait for subs in PeerEvents and fix stream.runSyncTest

* swarm: enforce waitkademlia for snapshot tests

* swarm: fixed syncer tests and snapshot_sync_test

* swarm: linting of simulation package

* swarm: address review comments

* swarm/network/stream: fix delivery_test bugs and refactor

* swarm/network/stream: addressed PR comments @janos

* swarm/network/stream: enforce waitKademlia, improve TestIntervals

* swarm/network/stream: TestIntervals not waiting for chunk to be stored

swarm/network/simulation/bucket.go

@@ -43,7 +43,7 @@ func (s *Simulation) SetNodeItem(id discover.NodeID, key interface{}, value inte
 	s.buckets[id].Store(key, value)
 }
 
-// NodeItems returns a map of items from all nodes that are all set under the
+// NodesItems returns a map of items from all nodes that are all set under the
 // same BucketKey.
 func (s *Simulation) NodesItems(key interface{}) (values map[discover.NodeID]interface{}) {
 	s.mu.RLock()

swarm/network/simulation/connect.go

@@ -54,7 +54,7 @@ func (s *Simulation) ConnectToLastNode(id discover.NodeID) (err error) {
 // ConnectToRandomNode connects the node with provieded NodeID
 // to a random node that is up.
 func (s *Simulation) ConnectToRandomNode(id discover.NodeID) (err error) {
-	n := s.randomUpNode(id)
+	n := s.RandomUpNode(id)
 	if n == nil {
 		return ErrNodeNotFound
 	}
@@ -135,7 +135,7 @@ func (s *Simulation) ConnectNodesStar(id discover.NodeID, ids []discover.NodeID)
 	return nil
 }
 
-// ConnectNodesStar connects all nodes in a star topology
+// ConnectNodesStarPivot connects all nodes in a star topology
 // with the center at already set pivot node.
 // If ids argument is nil, all nodes that are up will be connected.
 func (s *Simulation) ConnectNodesStarPivot(ids []discover.NodeID) (err error) {

swarm/network/simulation/events.go

@@ -18,6 +18,7 @@ package simulation
 
 import (
 	"context"
+	"sync"
 
 	"github.com/ethereum/go-ethereum/p2p/discover"
 
@@ -71,24 +72,32 @@ func (f *PeerEventsFilter) MsgCode(c uint64) *PeerEventsFilter {
 func (s *Simulation) PeerEvents(ctx context.Context, ids []discover.NodeID, filters ...*PeerEventsFilter) <-chan PeerEvent {
 	eventC := make(chan PeerEvent)
 
+	// wait group to make sure all subscriptions to admin peerEvents are established
+	// before this function returns.
+	var subsWG sync.WaitGroup
 	for _, id := range ids {
 		s.shutdownWG.Add(1)
+		subsWG.Add(1)
 		go func(id discover.NodeID) {
 			defer s.shutdownWG.Done()
 
 			client, err := s.Net.GetNode(id).Client()
 			if err != nil {
+				subsWG.Done()
 				eventC <- PeerEvent{NodeID: id, Error: err}
 				return
 			}
 			events := make(chan *p2p.PeerEvent)
 			sub, err := client.Subscribe(ctx, "admin", events, "peerEvents")
 			if err != nil {
+				subsWG.Done()
 				eventC <- PeerEvent{NodeID: id, Error: err}
 				return
 			}
 			defer sub.Unsubscribe()
 
+			subsWG.Done()
+
 			for {
 				select {
 				case <-ctx.Done():
@@ -153,5 +162,7 @@ func (s *Simulation) PeerEvents(ctx context.Context, ids []discover.NodeID, filt
 		}(id)
 	}
 
+	// wait all subscriptions
+	subsWG.Wait()
 	return eventC
 }

swarm/network/simulation/http.go

@@ -29,7 +29,7 @@ var (
 	DefaultHTTPSimAddr = ":8888"
 )
 
-//`With`(builder) pattern constructor for Simulation to
+//WithServer implements the builder pattern constructor for Simulation to
 //start with a HTTP server
 func (s *Simulation) WithServer(addr string) *Simulation {
 	//assign default addr if nothing provided
@@ -46,7 +46,12 @@ func (s *Simulation) WithServer(addr string) *Simulation {
 		Addr:    addr,
 		Handler: s.handler,
 	}
-	go s.httpSrv.ListenAndServe()
+	go func() {
+		err := s.httpSrv.ListenAndServe()
+		if err != nil {
+			log.Error("Error starting the HTTP server", "error", err)
+		}
+	}()
 	return s
 }
 
@@ -55,7 +60,7 @@ func (s *Simulation) addSimulationRoutes() {
 	s.handler.POST("/runsim", s.RunSimulation)
 }
 
-// StartNetwork starts all nodes in the network
+// RunSimulation is the actual POST endpoint runner
 func (s *Simulation) RunSimulation(w http.ResponseWriter, req *http.Request) {
 	log.Debug("RunSimulation endpoint running")
 	s.runC <- struct{}{}

swarm/network/simulation/http_test.go

@@ -96,7 +96,12 @@ func sendRunSignal(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Request failed: %v", err)
 	}
-	defer resp.Body.Close()
+	defer func() {
+		err := resp.Body.Close()
+		if err != nil {
+			log.Error("Error closing response body", "err", err)
+		}
+	}()
 	log.Debug("Signal sent")
 	if resp.StatusCode != http.StatusOK {
 		t.Fatalf("err %s", resp.Status)

swarm/network/simulation/node.go

@@ -195,7 +195,7 @@ func (s *Simulation) AddNodesAndConnectStar(count int, opts ...AddNodeOption) (i
 	return ids, nil
 }
 
-//Upload a snapshot
+//UploadSnapshot uploads a snapshot to the simulation
 //This method tries to open the json file provided, applies the config to all nodes
 //and then loads the snapshot into the Simulation network
 func (s *Simulation) UploadSnapshot(snapshotFile string, opts ...AddNodeOption) error {
@@ -203,7 +203,12 @@ func (s *Simulation) UploadSnapshot(snapshotFile string, opts ...AddNodeOption)
 	if err != nil {
 		return err
 	}
-	defer f.Close()
+	defer func() {
+		err := f.Close()
+		if err != nil {
+			log.Error("Error closing snapshot file", "err", err)
+		}
+	}()
 	jsonbyte, err := ioutil.ReadAll(f)
 	if err != nil {
 		return err
@@ -294,7 +299,7 @@ func (s *Simulation) StopNode(id discover.NodeID) (err error) {
 
 // StopRandomNode stops a random node.
 func (s *Simulation) StopRandomNode() (id discover.NodeID, err error) {
-	n := s.randomUpNode()
+	n := s.RandomUpNode()
 	if n == nil {
 		return id, ErrNodeNotFound
 	}
@@ -324,18 +329,18 @@ func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
-// randomUpNode returns a random SimNode that is up.
+// RandomUpNode returns a random SimNode that is up.
 // Arguments are NodeIDs for nodes that should not be returned.
-func (s *Simulation) randomUpNode(exclude ...discover.NodeID) *adapters.SimNode {
+func (s *Simulation) RandomUpNode(exclude ...discover.NodeID) *adapters.SimNode {
 	return s.randomNode(s.UpNodeIDs(), exclude...)
 }
 
-// randomUpNode returns a random SimNode that is not up.
+// randomDownNode returns a random SimNode that is not up.
 func (s *Simulation) randomDownNode(exclude ...discover.NodeID) *adapters.SimNode {
 	return s.randomNode(s.DownNodeIDs(), exclude...)
 }
 
-// randomUpNode returns a random SimNode from the slice of NodeIDs.
+// randomNode returns a random SimNode from the slice of NodeIDs.
 func (s *Simulation) randomNode(ids []discover.NodeID, exclude ...discover.NodeID) *adapters.SimNode {
 	for _, e := range exclude {
 		var i int

swarm/network/simulation/service.go

@@ -39,7 +39,7 @@ func (s *Simulation) Service(name string, id discover.NodeID) node.Service {
 // RandomService returns a single Service by name on a
 // randomly chosen node that is up.
 func (s *Simulation) RandomService(name string) node.Service {
-	n := s.randomUpNode()
+	n := s.RandomUpNode()
 	if n == nil {
 		return nil
 	}

swarm/network/stream/common_test.go

@@ -18,135 +18,70 @@ package stream
 
 import (
 	"context"
-	"encoding/binary"
+	crand "crypto/rand"
 	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"os"
+	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/discover"
-	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm/network"
+	"github.com/ethereum/go-ethereum/swarm/network/simulation"
+	"github.com/ethereum/go-ethereum/swarm/pot"
 	"github.com/ethereum/go-ethereum/swarm/state"
 	"github.com/ethereum/go-ethereum/swarm/storage"
-	"github.com/ethereum/go-ethereum/swarm/storage/mock"
-	"github.com/ethereum/go-ethereum/swarm/storage/mock/db"
+	mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
 	colorable "github.com/mattn/go-colorable"
 )
 
 var (
-	deliveries   map[discover.NodeID]*Delivery
-	stores       map[discover.NodeID]storage.ChunkStore
-	toAddr       func(discover.NodeID) *network.BzzAddr
-	peerCount    func(discover.NodeID) int
-	adapter      = flag.String("adapter", "sim", "type of simulation: sim|exec|docker")
 	loglevel     = flag.Int("loglevel", 2, "verbosity of logs")
 	nodes        = flag.Int("nodes", 0, "number of nodes")
 	chunks       = flag.Int("chunks", 0, "number of chunks")
 	useMockStore = flag.Bool("mockstore", false, "disabled mock store (default: enabled)")
-)
+	longrunning  = flag.Bool("longrunning", false, "do run long-running tests")
 
-var (
-	defaultSkipCheck  bool
-	waitPeerErrC      chan error
-	chunkSize         = 4096
-	registries        map[discover.NodeID]*TestRegistry
-	createStoreFunc   func(id discover.NodeID, addr *network.BzzAddr) (storage.ChunkStore, error)
-	getRetrieveFunc   = defaultRetrieveFunc
-	subscriptionCount = 0
-	globalStore       mock.GlobalStorer
-	globalStoreDir    string
-)
+	bucketKeyDB        = simulation.BucketKey("db")
+	bucketKeyStore     = simulation.BucketKey("store")
+	bucketKeyFileStore = simulation.BucketKey("filestore")
+	bucketKeyNetStore  = simulation.BucketKey("netstore")
+	bucketKeyDelivery  = simulation.BucketKey("delivery")
+	bucketKeyRegistry  = simulation.BucketKey("registry")
 
-var services = adapters.Services{
-	"streamer":          NewStreamerService,
-	"intervalsStreamer": newIntervalsStreamerService,
-}
+	chunkSize = 4096
+	pof       = pot.DefaultPof(256)
+)
 
 func init() {
 	flag.Parse()
-	// register the Delivery service which will run as a devp2p
-	// protocol when using the exec adapter
-	adapters.RegisterServices(services)
+	rand.Seed(time.Now().UnixNano())
 
 	log.PrintOrigins(true)
 	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
 }
 
-func createGlobalStore() {
-	var err error
-	globalStoreDir, err = ioutil.TempDir("", "global.store")
+func createGlobalStore() (string, *mockdb.GlobalStore, error) {
+	var globalStore *mockdb.GlobalStore
+	globalStoreDir, err := ioutil.TempDir("", "global.store")
 	if err != nil {
 		log.Error("Error initiating global store temp directory!", "err", err)
-		return
+		return "", nil, err
 	}
-	globalStore, err = db.NewGlobalStore(globalStoreDir)
+	globalStore, err = mockdb.NewGlobalStore(globalStoreDir)
 	if err != nil {
 		log.Error("Error initiating global store!", "err", err)
+		return "", nil, err
 	}
-}
-
-// NewStreamerService
-func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) {
-	var err error
-	id := ctx.Config.ID
-	addr := toAddr(id)
-	kad := network.NewKademlia(addr.Over(), network.NewKadParams())
-	stores[id], err = createStoreFunc(id, addr)
-	if err != nil {
-		return nil, err
-	}
-	store := stores[id].(*storage.LocalStore)
-	db := storage.NewDBAPI(store)
-	delivery := NewDelivery(kad, db)
-	deliveries[id] = delivery
-	r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{
-		SkipCheck:  defaultSkipCheck,
-		DoRetrieve: false,
-	})
-	RegisterSwarmSyncerServer(r, db)
-	RegisterSwarmSyncerClient(r, db)
-	go func() {
-		waitPeerErrC <- waitForPeers(r, 1*time.Second, peerCount(id))
-	}()
-	fileStore := storage.NewFileStore(storage.NewNetStore(store, getRetrieveFunc(id)), storage.NewFileStoreParams())
-	testRegistry := &TestRegistry{Registry: r, fileStore: fileStore}
-	registries[id] = testRegistry
-	return testRegistry, nil
-}
-
-func defaultRetrieveFunc(id discover.NodeID) func(ctx context.Context, chunk *storage.Chunk) error {
-	return nil
-}
-
-func datadirsCleanup() {
-	for _, id := range ids {
-		os.RemoveAll(datadirs[id])
-	}
-	if globalStoreDir != "" {
-		os.RemoveAll(globalStoreDir)
-	}
-}
-
-//local stores need to be cleaned up after the sim is done
-func localStoreCleanup() {
-	log.Info("Cleaning up...")
-	for _, id := range ids {
-		registries[id].Close()
-		stores[id].Close()
-	}
-	log.Info("Local store cleanup done")
+	return globalStoreDir, globalStore, nil
 }
 
 func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) {
@@ -174,9 +109,7 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora
 
 	db := storage.NewDBAPI(localStore)
 	delivery := NewDelivery(to, db)
-	streamer := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{
-		SkipCheck: defaultSkipCheck,
-	})
+	streamer := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), nil)
 	teardown := func() {
 		streamer.Close()
 		removeDataDir()
@@ -233,22 +166,6 @@ func (rrs *roundRobinStore) Close() {
 	}
 }
 
-type TestRegistry struct {
-	*Registry
-	fileStore *storage.FileStore
-}
-
-func (r *TestRegistry) APIs() []rpc.API {
-	a := r.Registry.APIs()
-	a = append(a, rpc.API{
-		Namespace: "stream",
-		Version:   "3.0",
-		Service:   r,
-		Public:    true,
-	})
-	return a
-}
-
 func readAll(fileStore *storage.FileStore, hash []byte) (int64, error) {
 	r, _ := fileStore.Retrieve(context.TODO(), hash)
 	buf := make([]byte, 1024)
@@ -265,185 +182,74 @@ func readAll(fileStore *storage.FileStore, hash []byte) (int64, error) {
 	return total, nil
 }
 
-func (r *TestRegistry) ReadAll(hash common.Hash) (int64, error) {
-	return readAll(r.fileStore, hash[:])
-}
-
-func (r *TestRegistry) Start(server *p2p.Server) error {
-	return r.Registry.Start(server)
-}
-
-func (r *TestRegistry) Stop() error {
-	return r.Registry.Stop()
-}
-
-type TestExternalRegistry struct {
-	*Registry
-}
-
-func (r *TestExternalRegistry) APIs() []rpc.API {
-	a := r.Registry.APIs()
-	a = append(a, rpc.API{
-		Namespace: "stream",
-		Version:   "3.0",
-		Service:   r,
-		Public:    true,
-	})
-	return a
-}
-
-func (r *TestExternalRegistry) GetHashes(ctx context.Context, peerId discover.NodeID, s Stream) (*rpc.Subscription, error) {
-	peer := r.getPeer(peerId)
-
-	client, err := peer.getClient(ctx, s)
-	if err != nil {
-		return nil, err
-	}
-
-	c := client.Client.(*testExternalClient)
+func uploadFilesToNodes(sim *simulation.Simulation) ([]storage.Address, []string, error) {
+	nodes := sim.UpNodeIDs()
+	nodeCnt := len(nodes)
+	log.Debug(fmt.Sprintf("Uploading %d files to nodes", nodeCnt))
+	//array holding generated files
+	rfiles := make([]string, nodeCnt)
+	//array holding the root hashes of the files
+	rootAddrs := make([]storage.Address, nodeCnt)
 
-	notifier, supported := rpc.NotifierFromContext(ctx)
-	if !supported {
-		return nil, fmt.Errorf("Subscribe not supported")
-	}
-
-	sub := notifier.CreateSubscription()
-
-	go func() {
-		// if we begin sending event immediately some events
-		// will probably be dropped since the subscription ID might not be send to
-		// the client.
-		// ref: rpc/subscription_test.go#L65
-		time.Sleep(1 * time.Second)
-		for {
-			select {
-			case h := <-c.hashes:
-				<-c.enableNotificationsC // wait for notification subscription to complete
-				if err := notifier.Notify(sub.ID, h); err != nil {
-					log.Warn(fmt.Sprintf("rpc sub notifier notify stream %s: %v", s, err))
-				}
-			case err := <-sub.Err():
-				if err != nil {
-					log.Warn(fmt.Sprintf("caught subscription error in stream %s: %v", s, err))
-				}
-			case <-notifier.Closed():
-				log.Trace(fmt.Sprintf("rpc sub notifier closed"))
-				return
-			}
+	var err error
+	//for every node, generate a file and upload
+	for i, id := range nodes {
+		item, ok := sim.NodeItem(id, bucketKeyFileStore)
+		if !ok {
+			return nil, nil, fmt.Errorf("Error accessing localstore")
 		}
-	}()
-
-	return sub, nil
-}
-
-func (r *TestExternalRegistry) EnableNotifications(peerId discover.NodeID, s Stream) error {
-	peer := r.getPeer(peerId)
-
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	client, err := peer.getClient(ctx, s)
-	if err != nil {
-		return err
-	}
-
-	close(client.Client.(*testExternalClient).enableNotificationsC)
-
-	return nil
-}
-
-// TODO: merge functionalities of testExternalClient and testExternalServer
-// with testClient and testServer.
-
-type testExternalClient struct {
-	hashes               chan []byte
-	db                   *storage.DBAPI
-	enableNotificationsC chan struct{}
-}
-
-func newTestExternalClient(db *storage.DBAPI) *testExternalClient {
-	return &testExternalClient{
-		hashes:               make(chan []byte),
-		db:                   db,
-		enableNotificationsC: make(chan struct{}),
-	}
-}
-
-func (c *testExternalClient) NeedData(ctx context.Context, hash []byte) func() {
-	chunk, _ := c.db.GetOrCreateRequest(ctx, hash)
-	if chunk.ReqC == nil {
-		return nil
-	}
-	c.hashes <- hash
-	return func() {
-		chunk.WaitToStore()
+		fileStore := item.(*storage.FileStore)
+		//generate a file
+		rfiles[i], err = generateRandomFile()
+		if err != nil {
+			return nil, nil, err
+		}
+		//store it (upload it) on the FileStore
+		ctx := context.TODO()
+		rk, wait, err := fileStore.Store(ctx, strings.NewReader(rfiles[i]), int64(len(rfiles[i])), false)
+		log.Debug("Uploaded random string file to node")
+		if err != nil {
+			return nil, nil, err
+		}
+		err = wait(ctx)
+		if err != nil {
+			return nil, nil, err
+		}
+		rootAddrs[i] = rk
 	}
+	return rootAddrs, rfiles, nil
 }
 
-func (c *testExternalClient) BatchDone(Stream, uint64, []byte, []byte) func() (*TakeoverProof, error) {
-	return nil
-}
-
-func (c *testExternalClient) Close() {}
-
-const testExternalServerBatchSize = 10
-
-type testExternalServer struct {
-	t         string
-	keyFunc   func(key []byte, index uint64)
-	sessionAt uint64
-	maxKeys   uint64
-	streamer  *TestExternalRegistry
-}
-
-func newTestExternalServer(t string, sessionAt, maxKeys uint64, keyFunc func(key []byte, index uint64)) *testExternalServer {
-	if keyFunc == nil {
-		keyFunc = binary.BigEndian.PutUint64
-	}
-	return &testExternalServer{
-		t:         t,
-		keyFunc:   keyFunc,
-		sessionAt: sessionAt,
-		maxKeys:   maxKeys,
+//generate a random file (string)
+func generateRandomFile() (string, error) {
+	//generate a random file size between minFileSize and maxFileSize
+	fileSize := rand.Intn(maxFileSize-minFileSize) + minFileSize
+	log.Debug(fmt.Sprintf("Generated file with filesize %d kB", fileSize))
+	b := make([]byte, fileSize*1024)
+	_, err := crand.Read(b)
+	if err != nil {
+		log.Error("Error generating random file.", "err", err)
+		return "", err
 	}
+	return string(b), nil
 }
 
-func (s *testExternalServer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) {
-	if from == 0 && to == 0 {
-		from = s.sessionAt
-		to = s.sessionAt + testExternalServerBatchSize
-	}
-	if to-from > testExternalServerBatchSize {
-		to = from + testExternalServerBatchSize - 1
-	}
-	if from >= s.maxKeys && to > s.maxKeys {
-		return nil, 0, 0, nil, io.EOF
-	}
-	if to > s.maxKeys {
-		to = s.maxKeys
+//create a local store for the given node
+func createTestLocalStorageForID(id discover.NodeID, addr *network.BzzAddr) (storage.ChunkStore, string, error) {
+	var datadir string
+	var err error
+	datadir, err = ioutil.TempDir("", fmt.Sprintf("syncer-test-%s", id.TerminalString()))
+	if err != nil {
+		return nil, "", err
 	}
-	b := make([]byte, HashSize*(to-from+1))
-	for i := from; i <= to; i++ {
-		s.keyFunc(b[(i-from)*HashSize:(i-from+1)*HashSize], i)
+	var store storage.ChunkStore
+	params := storage.NewDefaultLocalStoreParams()
+	params.ChunkDbPath = datadir
+	params.BaseKey = addr.Over()
+	store, err = storage.NewTestLocalStoreForAddr(params)
+	if err != nil {
+		os.RemoveAll(datadir)
+		return nil, "", err
 	}
-	return b, from, to, nil, nil
-}
-
-func (s *testExternalServer) GetData(context.Context, []byte) ([]byte, error) {
-	return make([]byte, 4096), nil
-}
-
-func (s *testExternalServer) Close() {}
-
-// Sets the global value defaultSkipCheck.
-// It should be used in test function defer to reset the global value
-// to the original value.
-//
-// defer setDefaultSkipCheck(defaultSkipCheck)
-// defaultSkipCheck = skipCheck
-//
-// This works as defer function arguments evaluations are evaluated as ususal,
-// but only the function body invocation is deferred.
-func setDefaultSkipCheck(skipCheck bool) {
-	defaultSkipCheck = skipCheck
+	return store, datadir, nil
 }

swarm/network/stream/intervals_test.go

@@ -22,52 +22,22 @@ import (
 	"encoding/binary"
 	"fmt"
 	"io"
+	"os"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/discover"
-	"github.com/ethereum/go-ethereum/p2p/simulations"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm/network"
-	streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing"
+	"github.com/ethereum/go-ethereum/swarm/network/simulation"
 	"github.com/ethereum/go-ethereum/swarm/state"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
-var (
-	externalStreamName             = "externalStream"
-	externalStreamSessionAt uint64 = 50
-	externalStreamMaxKeys   uint64 = 100
-)
-
-func newIntervalsStreamerService(ctx *adapters.ServiceContext) (node.Service, error) {
-	id := ctx.Config.ID
-	addr := toAddr(id)
-	kad := network.NewKademlia(addr.Over(), network.NewKadParams())
-	store := stores[id].(*storage.LocalStore)
-	db := storage.NewDBAPI(store)
-	delivery := NewDelivery(kad, db)
-	deliveries[id] = delivery
-	r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{
-		SkipCheck: defaultSkipCheck,
-	})
-
-	r.RegisterClientFunc(externalStreamName, func(p *Peer, t string, live bool) (Client, error) {
-		return newTestExternalClient(db), nil
-	})
-	r.RegisterServerFunc(externalStreamName, func(p *Peer, t string, live bool) (Server, error) {
-		return newTestExternalServer(t, externalStreamSessionAt, externalStreamMaxKeys, nil), nil
-	})
-
-	go func() {
-		waitPeerErrC <- waitForPeers(r, 1*time.Second, peerCount(id))
-	}()
-	return &TestExternalRegistry{r}, nil
-}
-
 func TestIntervals(t *testing.T) {
 	testIntervals(t, true, nil, false)
 	testIntervals(t, false, NewRange(9, 26), false)
@@ -81,237 +51,337 @@ func TestIntervals(t *testing.T) {
 func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) {
 	nodes := 2
 	chunkCount := dataChunkCount
+	externalStreamName := "externalStream"
+	externalStreamSessionAt := uint64(50)
+	externalStreamMaxKeys := uint64(100)
 
-	defer setDefaultSkipCheck(defaultSkipCheck)
-	defaultSkipCheck = skipCheck
-
-	toAddr = network.NewAddrFromNodeID
-	conf := &streamTesting.RunConfig{
-		Adapter:        *adapter,
-		NodeCount:      nodes,
-		ConnLevel:      1,
-		ToAddr:         toAddr,
-		Services:       services,
-		DefaultService: "intervalsStreamer",
-	}
+	sim := simulation.New(map[string]simulation.ServiceFunc{
+		"intervalsStreamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
 
-	sim, teardown, err := streamTesting.NewSimulation(conf)
-	var rpcSubscriptionsWg sync.WaitGroup
-	defer func() {
-		rpcSubscriptionsWg.Wait()
-		teardown()
-	}()
-	if err != nil {
-		t.Fatal(err)
-	}
+			id := ctx.Config.ID
+			addr := network.NewAddrFromNodeID(id)
+			store, datadir, err := createTestLocalStorageForID(id, addr)
+			if err != nil {
+				return nil, nil, err
+			}
+			bucket.Store(bucketKeyStore, store)
+			cleanup = func() {
+				store.Close()
+				os.RemoveAll(datadir)
+			}
+			localStore := store.(*storage.LocalStore)
+			db := storage.NewDBAPI(localStore)
+			kad := network.NewKademlia(addr.Over(), network.NewKadParams())
+			delivery := NewDelivery(kad, db)
 
-	stores = make(map[discover.NodeID]storage.ChunkStore)
-	deliveries = make(map[discover.NodeID]*Delivery)
-	for i, id := range sim.IDs {
-		stores[id] = sim.Stores[i]
-	}
+			r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{
+				SkipCheck: skipCheck,
+			})
+			bucket.Store(bucketKeyRegistry, r)
 
-	peerCount = func(id discover.NodeID) int {
-		return 1
-	}
+			r.RegisterClientFunc(externalStreamName, func(p *Peer, t string, live bool) (Client, error) {
+				return newTestExternalClient(db), nil
+			})
+			r.RegisterServerFunc(externalStreamName, func(p *Peer, t string, live bool) (Server, error) {
+				return newTestExternalServer(t, externalStreamSessionAt, externalStreamMaxKeys, nil), nil
+			})
 
-	fileStore := storage.NewFileStore(sim.Stores[0], storage.NewFileStoreParams())
-	size := chunkCount * chunkSize
-	ctx := context.TODO()
-	_, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
-	if err != nil {
-		t.Fatal(err)
-	}
-	err = wait(ctx)
+			fileStore := storage.NewFileStore(localStore, storage.NewFileStoreParams())
+			bucket.Store(bucketKeyFileStore, fileStore)
+
+			return r, cleanup, nil
+
+		},
+	})
+	defer sim.Close()
+
+	log.Info("Adding nodes to simulation")
+	_, err := sim.AddNodesAndConnectChain(nodes)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	errc := make(chan error, 1)
-	waitPeerErrC = make(chan error)
-	quitC := make(chan struct{})
-	defer close(quitC)
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Second)
+	defer cancel()
 
-	action := func(ctx context.Context) error {
-		i := 0
-		for err := range waitPeerErrC {
-			if err != nil {
-				return fmt.Errorf("error waiting for peers: %s", err)
-			}
-			i++
-			if i == nodes {
-				break
-			}
+	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
+		nodeIDs := sim.UpNodeIDs()
+		storer := nodeIDs[0]
+		checker := nodeIDs[1]
+
+		item, ok := sim.NodeItem(storer, bucketKeyFileStore)
+		if !ok {
+			return fmt.Errorf("No filestore")
 		}
+		fileStore := item.(*storage.FileStore)
 
-		id := sim.IDs[1]
+		size := chunkCount * chunkSize
+		_, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+		if err != nil {
+			log.Error("Store error: %v", "err", err)
+			t.Fatal(err)
+		}
+		err = wait(ctx)
+		if err != nil {
+			log.Error("Wait error: %v", "err", err)
+			t.Fatal(err)
+		}
 
-		err := sim.CallClient(id, func(client *rpc.Client) error {
+		item, ok = sim.NodeItem(checker, bucketKeyRegistry)
+		if !ok {
+			return fmt.Errorf("No registry")
+		}
+		registry := item.(*Registry)
 
-			sid := sim.IDs[0]
+		liveErrC := make(chan error)
+		historyErrC := make(chan error)
 
-			doneC, err := streamTesting.WatchDisconnections(id, client, errc, quitC)
-			if err != nil {
-				return err
+		if _, err := sim.WaitTillHealthy(ctx, 2); err != nil {
+			log.Error("WaitKademlia error: %v", "err", err)
+			return err
+		}
+
+		log.Debug("Watching for disconnections")
+		disconnections := sim.PeerEvents(
+			context.Background(),
+			sim.NodeIDs(),
+			simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+		)
+
+		go func() {
+			for d := range disconnections {
+				if d.Error != nil {
+					log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+					t.Fatal(d.Error)
+				}
 			}
-			rpcSubscriptionsWg.Add(1)
-			go func() {
-				<-doneC
-				rpcSubscriptionsWg.Done()
+		}()
+
+		go func() {
+			if !live {
+				close(liveErrC)
+				return
+			}
+
+			var err error
+			defer func() {
+				liveErrC <- err
 			}()
-			ctx, cancel := context.WithTimeout(ctx, 100*time.Second)
-			defer cancel()
 
-			err = client.CallContext(ctx, nil, "stream_subscribeStream", sid, NewStream(externalStreamName, "", live), history, Top)
+			// live stream
+			var liveHashesChan chan []byte
+			liveHashesChan, err = getHashes(ctx, registry, storer, NewStream(externalStreamName, "", true))
 			if err != nil {
-				return err
+				log.Error("Subscription error: %v", "err", err)
+				return
 			}
+			i := externalStreamSessionAt
 
-			liveErrC := make(chan error)
-			historyErrC := make(chan error)
+			// we have subscribed, enable notifications
+			err = enableNotifications(registry, storer, NewStream(externalStreamName, "", true))
+			if err != nil {
+				return
+			}
 
-			go func() {
-				if !live {
-					close(liveErrC)
+			for {
+				select {
+				case hash := <-liveHashesChan:
+					h := binary.BigEndian.Uint64(hash)
+					if h != i {
+						err = fmt.Errorf("expected live hash %d, got %d", i, h)
+						return
+					}
+					i++
+					if i > externalStreamMaxKeys {
+						return
+					}
+				case <-ctx.Done():
 					return
 				}
+			}
+		}()
 
-				var err error
-				defer func() {
-					liveErrC <- err
-				}()
+		go func() {
+			if live && history == nil {
+				close(historyErrC)
+				return
+			}
 
-				// live stream
-				liveHashesChan := make(chan []byte)
-				liveSubscription, err := client.Subscribe(ctx, "stream", liveHashesChan, "getHashes", sid, NewStream(externalStreamName, "", true))
-				if err != nil {
-					return
-				}
-				defer liveSubscription.Unsubscribe()
+			var err error
+			defer func() {
+				historyErrC <- err
+			}()
 
-				i := externalStreamSessionAt
+			// history stream
+			var historyHashesChan chan []byte
+			historyHashesChan, err = getHashes(ctx, registry, storer, NewStream(externalStreamName, "", false))
+			if err != nil {
+				return
+			}
 
-				// we have subscribed, enable notifications
-				err = client.CallContext(ctx, nil, "stream_enableNotifications", sid, NewStream(externalStreamName, "", true))
-				if err != nil {
-					return
+			var i uint64
+			historyTo := externalStreamMaxKeys
+			if history != nil {
+				i = history.From
+				if history.To != 0 {
+					historyTo = history.To
 				}
+			}
 
-				for {
-					select {
-					case hash := <-liveHashesChan:
-						h := binary.BigEndian.Uint64(hash)
-						if h != i {
-							err = fmt.Errorf("expected live hash %d, got %d", i, h)
-							return
-						}
-						i++
-						if i > externalStreamMaxKeys {
-							return
-						}
-					case err = <-liveSubscription.Err():
+			// we have subscribed, enable notifications
+			err = enableNotifications(registry, storer, NewStream(externalStreamName, "", false))
+			if err != nil {
+				return
+			}
+
+			for {
+				select {
+				case hash := <-historyHashesChan:
+					h := binary.BigEndian.Uint64(hash)
+					if h != i {
+						err = fmt.Errorf("expected history hash %d, got %d", i, h)
 						return
-					case <-ctx.Done():
+					}
+					i++
+					if i > historyTo {
 						return
 					}
-				}
-			}()
-
-			go func() {
-				if live && history == nil {
-					close(historyErrC)
+				case <-ctx.Done():
 					return
 				}
+			}
+		}()
 
-				var err error
-				defer func() {
-					historyErrC <- err
-				}()
+		err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top)
+		if err != nil {
+			return err
+		}
+		if err := <-liveErrC; err != nil {
+			return err
+		}
+		if err := <-historyErrC; err != nil {
+			return err
+		}
 
-				// history stream
-				historyHashesChan := make(chan []byte)
-				historySubscription, err := client.Subscribe(ctx, "stream", historyHashesChan, "getHashes", sid, NewStream(externalStreamName, "", false))
-				if err != nil {
-					return
-				}
-				defer historySubscription.Unsubscribe()
-
-				var i uint64
-				historyTo := externalStreamMaxKeys
-				if history != nil {
-					i = history.From
-					if history.To != 0 {
-						historyTo = history.To
-					}
-				}
+		return nil
+	})
 
-				// we have subscribed, enable notifications
-				err = client.CallContext(ctx, nil, "stream_enableNotifications", sid, NewStream(externalStreamName, "", false))
-				if err != nil {
-					return
-				}
+	if result.Error != nil {
+		t.Fatal(result.Error)
+	}
+}
 
-				for {
-					select {
-					case hash := <-historyHashesChan:
-						h := binary.BigEndian.Uint64(hash)
-						if h != i {
-							err = fmt.Errorf("expected history hash %d, got %d", i, h)
-							return
-						}
-						i++
-						if i > historyTo {
-							return
-						}
-					case err = <-historySubscription.Err():
-						return
-					case <-ctx.Done():
-						return
-					}
-				}
-			}()
+func getHashes(ctx context.Context, r *Registry, peerID discover.NodeID, s Stream) (chan []byte, error) {
+	peer := r.getPeer(peerID)
 
-			if err := <-liveErrC; err != nil {
-				return err
-			}
-			if err := <-historyErrC; err != nil {
-				return err
-			}
+	client, err := peer.getClient(ctx, s)
+	if err != nil {
+		return nil, err
+	}
+
+	c := client.Client.(*testExternalClient)
+
+	return c.hashes, nil
+}
+
+func enableNotifications(r *Registry, peerID discover.NodeID, s Stream) error {
+	peer := r.getPeer(peerID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
 
-			return nil
-		})
+	client, err := peer.getClient(ctx, s)
+	if err != nil {
 		return err
 	}
-	check := func(ctx context.Context, id discover.NodeID) (bool, error) {
-		select {
-		case err := <-errc:
-			return false, err
-		case <-ctx.Done():
-			return false, ctx.Err()
-		default:
+
+	close(client.Client.(*testExternalClient).enableNotificationsC)
+
+	return nil
+}
+
+type testExternalClient struct {
+	hashes               chan []byte
+	db                   *storage.DBAPI
+	enableNotificationsC chan struct{}
+}
+
+func newTestExternalClient(db *storage.DBAPI) *testExternalClient {
+	return &testExternalClient{
+		hashes:               make(chan []byte),
+		db:                   db,
+		enableNotificationsC: make(chan struct{}),
+	}
+}
+
+func (c *testExternalClient) NeedData(ctx context.Context, hash []byte) func() {
+	chunk, _ := c.db.GetOrCreateRequest(ctx, hash)
+	if chunk.ReqC == nil {
+		return nil
+	}
+	c.hashes <- hash
+	//NOTE: This was failing on go1.9.x with a deadlock.
+	//Sometimes this function would just block
+	//It is commented now, but it may be well worth after the chunk refactor
+	//to re-enable this and see if the problem has been addressed
+	/*
+		return func() {
+			return chunk.WaitToStore()
 		}
-		return true, nil
+	*/
+	return nil
+}
+
+func (c *testExternalClient) BatchDone(Stream, uint64, []byte, []byte) func() (*TakeoverProof, error) {
+	return nil
+}
+
+func (c *testExternalClient) Close() {}
+
+const testExternalServerBatchSize = 10
+
+type testExternalServer struct {
+	t         string
+	keyFunc   func(key []byte, index uint64)
+	sessionAt uint64
+	maxKeys   uint64
+}
+
+func newTestExternalServer(t string, sessionAt, maxKeys uint64, keyFunc func(key []byte, index uint64)) *testExternalServer {
+	if keyFunc == nil {
+		keyFunc = binary.BigEndian.PutUint64
 	}
+	return &testExternalServer{
+		t:         t,
+		keyFunc:   keyFunc,
+		sessionAt: sessionAt,
+		maxKeys:   maxKeys,
+	}
+}
 
-	conf.Step = &simulations.Step{
-		Action:  action,
-		Trigger: streamTesting.Trigger(10*time.Millisecond, quitC, sim.IDs[0]),
-		Expect: &simulations.Expectation{
-			Nodes: sim.IDs[1:1],
-			Check: check,
-		},
+func (s *testExternalServer) SetNextBatch(from uint64, to uint64) ([]byte, uint64, uint64, *HandoverProof, error) {
+	if from == 0 && to == 0 {
+		from = s.sessionAt
+		to = s.sessionAt + testExternalServerBatchSize
 	}
-	startedAt := time.Now()
-	timeout := 300 * time.Second
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-	result, err := sim.Run(ctx, conf)
-	finishedAt := time.Now()
-	if err != nil {
-		t.Fatalf("Setting up simulation failed: %v", err)
+	if to-from > testExternalServerBatchSize {
+		to = from + testExternalServerBatchSize - 1
 	}
-	if result.Error != nil {
-		t.Fatalf("Simulation failed: %s", result.Error)
+	if from >= s.maxKeys && to > s.maxKeys {
+		return nil, 0, 0, nil, io.EOF
+	}
+	if to > s.maxKeys {
+		to = s.maxKeys
+	}
+	b := make([]byte, HashSize*(to-from+1))
+	for i := from; i <= to; i++ {
+		s.keyFunc(b[(i-from)*HashSize:(i-from+1)*HashSize], i)
 	}
-	streamTesting.CheckResult(t, result, startedAt, finishedAt)
+	return b, from, to, nil, nil
 }
+
+func (s *testExternalServer) GetData(context.Context, []byte) ([]byte, error) {
+	return make([]byte, 4096), nil
+}
+
+func (s *testExternalServer) Close() {}

swarm/network/stream/syncer_test.go

@@ -23,18 +23,22 @@ import (
 	"io"
 	"io/ioutil"
 	"math"
+	"os"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/p2p/discover"
-	"github.com/ethereum/go-ethereum/p2p/simulations"
-	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/swarm/log"
 	"github.com/ethereum/go-ethereum/swarm/network"
-	streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing"
+	"github.com/ethereum/go-ethereum/swarm/network/simulation"
+	"github.com/ethereum/go-ethereum/swarm/state"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	mockdb "github.com/ethereum/go-ethereum/swarm/storage/mock/db"
 )
 
 const dataChunkCount = 200
@@ -46,222 +50,193 @@ func TestSyncerSimulation(t *testing.T) {
 	testSyncBetweenNodes(t, 16, 1, dataChunkCount, true, 1)
 }
 
-func createMockStore(id discover.NodeID, addr *network.BzzAddr) (storage.ChunkStore, error) {
-	var err error
+func createMockStore(globalStore *mockdb.GlobalStore, id discover.NodeID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) {
 	address := common.BytesToAddress(id.Bytes())
 	mockStore := globalStore.NewNodeStore(address)
 	params := storage.NewDefaultLocalStoreParams()
-	datadirs[id], err = ioutil.TempDir("", "localMockStore-"+id.TerminalString())
+
+	datadir, err = ioutil.TempDir("", "localMockStore-"+id.TerminalString())
 	if err != nil {
-		return nil, err
+		return nil, "", err
 	}
-	params.Init(datadirs[id])
+	params.Init(datadir)
 	params.BaseKey = addr.Over()
-	lstore, err := storage.NewLocalStore(params, mockStore)
-	return lstore, nil
+	lstore, err = storage.NewLocalStore(params, mockStore)
+	return lstore, datadir, nil
 }
 
 func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool, po uint8) {
-	defer setDefaultSkipCheck(defaultSkipCheck)
-	defaultSkipCheck = skipCheck
-	//data directories for each node and store
-	datadirs = make(map[discover.NodeID]string)
-	if *useMockStore {
-		createStoreFunc = createMockStore
-		createGlobalStore()
-	} else {
-		createStoreFunc = createTestLocalStorageFromSim
-	}
-	defer datadirsCleanup()
+	sim := simulation.New(map[string]simulation.ServiceFunc{
+		"streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) {
+			var store storage.ChunkStore
+			var globalStore *mockdb.GlobalStore
+			var gDir, datadir string
+
+			id := ctx.Config.ID
+			addr := network.NewAddrFromNodeID(id)
+			//hack to put addresses in same space
+			addr.OAddr[0] = byte(0)
+
+			if *useMockStore {
+				gDir, globalStore, err = createGlobalStore()
+				if err != nil {
+					return nil, nil, fmt.Errorf("Something went wrong; using mockStore enabled but globalStore is nil")
+				}
+				store, datadir, err = createMockStore(globalStore, id, addr)
+			} else {
+				store, datadir, err = createTestLocalStorageForID(id, addr)
+			}
+			if err != nil {
+				return nil, nil, err
+			}
+			bucket.Store(bucketKeyStore, store)
+			cleanup = func() {
+				store.Close()
+				os.RemoveAll(datadir)
+				if *useMockStore {
+					err := globalStore.Close()
+					if err != nil {
+						log.Error("Error closing global store! %v", "err", err)
+					}
+					os.RemoveAll(gDir)
+				}
+			}
+			localStore := store.(*storage.LocalStore)
+			db := storage.NewDBAPI(localStore)
+			bucket.Store(bucketKeyDB, db)
+			kad := network.NewKademlia(addr.Over(), network.NewKadParams())
+			delivery := NewDelivery(kad, db)
+			bucket.Store(bucketKeyDelivery, delivery)
+
+			r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{
+				SkipCheck: skipCheck,
+			})
+
+			fileStore := storage.NewFileStore(storage.NewNetStore(localStore, nil), storage.NewFileStoreParams())
+			bucket.Store(bucketKeyFileStore, fileStore)
+
+			return r, cleanup, nil
+
+		},
+	})
+	defer sim.Close()
 
-	registries = make(map[discover.NodeID]*TestRegistry)
-	toAddr = func(id discover.NodeID) *network.BzzAddr {
-		addr := network.NewAddrFromNodeID(id)
-		//hack to put addresses in same space
-		addr.OAddr[0] = byte(0)
-		return addr
-	}
-	conf := &streamTesting.RunConfig{
-		Adapter:         *adapter,
-		NodeCount:       nodes,
-		ConnLevel:       conns,
-		ToAddr:          toAddr,
-		Services:        services,
-		EnableMsgEvents: false,
-	}
-	// HACK: these are global variables in the test so that they are available for
-	// the service constructor function
-	// TODO: will this work with exec/docker adapter?
-	// localstore of nodes made available for action and check calls
-	stores = make(map[discover.NodeID]storage.ChunkStore)
-	deliveries = make(map[discover.NodeID]*Delivery)
 	// create context for simulation run
 	timeout := 30 * time.Second
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
 	// defer cancel should come before defer simulation teardown
 	defer cancel()
 
-	// create simulation network with the config
-	sim, teardown, err := streamTesting.NewSimulation(conf)
-	var rpcSubscriptionsWg sync.WaitGroup
-	defer func() {
-		rpcSubscriptionsWg.Wait()
-		teardown()
-	}()
+	_, err := sim.AddNodesAndConnectChain(nodes)
 	if err != nil {
-		t.Fatal(err.Error())
+		t.Fatal(err)
 	}
+	result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error {
+		nodeIDs := sim.UpNodeIDs()
 
-	nodeIndex := make(map[discover.NodeID]int)
-	for i, id := range sim.IDs {
-		nodeIndex[id] = i
-		if !*useMockStore {
-			stores[id] = sim.Stores[i]
-			sim.Stores[i] = stores[id]
+		nodeIndex := make(map[discover.NodeID]int)
+		for i, id := range nodeIDs {
+			nodeIndex[id] = i
 		}
-	}
-	// peerCount function gives the number of peer connections for a nodeID
-	// this is needed for the service run function to wait until
-	// each protocol  instance runs and the streamer peers are available
-	peerCount = func(id discover.NodeID) int {
-		if sim.IDs[0] == id || sim.IDs[nodes-1] == id {
-			return 1
-		}
-		return 2
-	}
-	waitPeerErrC = make(chan error)
 
-	// create DBAPI-s for all nodes
-	dbs := make([]*storage.DBAPI, nodes)
-	for i := 0; i < nodes; i++ {
-		dbs[i] = storage.NewDBAPI(sim.Stores[i].(*storage.LocalStore))
-	}
-
-	// collect hashes in po 1 bin for each node
-	hashes := make([][]storage.Address, nodes)
-	totalHashes := 0
-	hashCounts := make([]int, nodes)
-	for i := nodes - 1; i >= 0; i-- {
-		if i < nodes-1 {
-			hashCounts[i] = hashCounts[i+1]
-		}
-		dbs[i].Iterator(0, math.MaxUint64, po, func(addr storage.Address, index uint64) bool {
-			hashes[i] = append(hashes[i], addr)
-			totalHashes++
-			hashCounts[i]++
-			return true
-		})
-	}
-
-	// errc is error channel for simulation
-	errc := make(chan error, 1)
-	quitC := make(chan struct{})
-	defer close(quitC)
+		disconnections := sim.PeerEvents(
+			context.Background(),
+			sim.NodeIDs(),
+			simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop),
+		)
+
+		go func() {
+			for d := range disconnections {
+				if d.Error != nil {
+					log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer)
+					t.Fatal(d.Error)
+				}
+			}
+		}()
 
-	// action is subscribe
-	action := func(ctx context.Context) error {
-		// need to wait till an aynchronous process registers the peers in streamer.peers
-		// that is used by Subscribe
-		// the global peerCount function tells how many connections each node has
-		// TODO: this is to be reimplemented with peerEvent watcher without global var
-		i := 0
-		for err := range waitPeerErrC {
+		// each node Subscribes to each other's swarmChunkServerStreamName
+		for j := 0; j < nodes-1; j++ {
+			id := nodeIDs[j]
+			client, err := sim.Net.GetNode(id).Client()
 			if err != nil {
-				return fmt.Errorf("error waiting for peers: %s", err)
+				t.Fatal(err)
 			}
-			i++
-			if i == nodes {
-				break
+			sid := nodeIDs[j+1]
+			client.CallContext(ctx, nil, "stream_subscribeStream", sid, NewStream("SYNC", FormatSyncBinKey(1), false), NewRange(0, 0), Top)
+			if err != nil {
+				return err
 			}
-		}
-		// each node Subscribes to each other's swarmChunkServerStreamName
-		for j := 0; j < nodes-1; j++ {
-			id := sim.IDs[j]
-			sim.Stores[j] = stores[id]
-			err := sim.CallClient(id, func(client *rpc.Client) error {
-				// report disconnect events to the error channel cos peers should not disconnect
-				doneC, err := streamTesting.WatchDisconnections(id, client, errc, quitC)
+			if j > 0 || nodes == 2 {
+				item, ok := sim.NodeItem(nodeIDs[j], bucketKeyFileStore)
+				if !ok {
+					return fmt.Errorf("No filestore")
+				}
+				fileStore := item.(*storage.FileStore)
+				size := chunkCount * chunkSize
+				_, wait, err := fileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
 				if err != nil {
-					return err
+					t.Fatal(err.Error())
 				}
-				rpcSubscriptionsWg.Add(1)
-				go func() {
-					<-doneC
-					rpcSubscriptionsWg.Done()
-				}()
-				ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
-				defer cancel()
-				// start syncing, i.e., subscribe to upstream peers po 1 bin
-				sid := sim.IDs[j+1]
-				return client.CallContext(ctx, nil, "stream_subscribeStream", sid, NewStream("SYNC", FormatSyncBinKey(1), false), NewRange(0, 0), Top)
-			})
-			if err != nil {
-				return err
+				wait(ctx)
 			}
 		}
 		// here we distribute chunks of a random file into stores 1...nodes
-		rrFileStore := storage.NewFileStore(newRoundRobinStore(sim.Stores[1:]...), storage.NewFileStoreParams())
-		size := chunkCount * chunkSize
-		_, wait, err := rrFileStore.Store(ctx, io.LimitReader(crand.Reader, int64(size)), int64(size), false)
-		if err != nil {
-			t.Fatal(err.Error())
-		}
-		// need to wait cos we then immediately collect the relevant bin content
-		wait(ctx)
-		if err != nil {
-			t.Fatal(err.Error())
+		if _, err := sim.WaitTillHealthy(ctx, 2); err != nil {
+			return err
 		}
 
-		return nil
-	}
-
-	// this makes sure check is not called before the previous call finishes
-	check := func(ctx context.Context, id discover.NodeID) (bool, error) {
-		select {
-		case err := <-errc:
-			return false, err
-		case <-ctx.Done():
-			return false, ctx.Err()
-		default:
+		// collect hashes in po 1 bin for each node
+		hashes := make([][]storage.Address, nodes)
+		totalHashes := 0
+		hashCounts := make([]int, nodes)
+		for i := nodes - 1; i >= 0; i-- {
+			if i < nodes-1 {
+				hashCounts[i] = hashCounts[i+1]
+			}
+			item, ok := sim.NodeItem(nodeIDs[i], bucketKeyDB)
+			if !ok {
+				return fmt.Errorf("No DB")
+			}
+			db := item.(*storage.DBAPI)
+			db.Iterator(0, math.MaxUint64, po, func(addr storage.Address, index uint64) bool {
+				hashes[i] = append(hashes[i], addr)
+				totalHashes++
+				hashCounts[i]++
+				return true
+			})
 		}
-
-		i := nodeIndex[id]
 		var total, found int
-
-		for j := i; j < nodes; j++ {
-			total += len(hashes[j])
-			for _, key := range hashes[j] {
-				chunk, err := dbs[i].Get(ctx, key)
-				if err == storage.ErrFetching {
-					<-chunk.ReqC
-				} else if err != nil {
-					continue
+		for _, node := range nodeIDs {
+			i := nodeIndex[node]
+
+			for j := i; j < nodes; j++ {
+				total += len(hashes[j])
+				for _, key := range hashes[j] {
+					item, ok := sim.NodeItem(nodeIDs[j], bucketKeyDB)
+					if !ok {
+						return fmt.Errorf("No DB")
+					}
+					db := item.(*storage.DBAPI)
+					chunk, err := db.Get(ctx, key)
+					if err == storage.ErrFetching {
+						<-chunk.ReqC
+					} else if err != nil {
+						continue
+					}
+					// needed for leveldb not to be closed?
+					// chunk.WaitToStore()
+					found++
 				}
-				// needed for leveldb not to be closed?
-				// chunk.WaitToStore()
-				found++
 			}
+			log.Debug("sync check", "node", node, "index", i, "bin", po, "found", found, "total", total)
 		}
-		log.Debug("sync check", "node", id, "index", i, "bin", po, "found", found, "total", total)
-		return total == found, nil
-	}
+		if total == found && total > 0 {
+			return nil
+		}
+		return fmt.Errorf("Total not equallying found: total is %d", total)
+	})
 
-	conf.Step = &simulations.Step{
-		Action:  action,
-		Trigger: streamTesting.Trigger(500*time.Millisecond, quitC, sim.IDs[0:nodes-1]...),
-		Expect: &simulations.Expectation{
-			Nodes: sim.IDs[0:1],
-			Check: check,
-		},
-	}
-	startedAt := time.Now()
-	result, err := sim.Run(ctx, conf)
-	finishedAt := time.Now()
-	if err != nil {
-		t.Fatalf("Setting up simulation failed: %v", err)
-	}
 	if result.Error != nil {
-		t.Fatalf("Simulation failed: %s", result.Error)
+		t.Fatal(result.Error)
 	}
-	streamTesting.CheckResult(t, result, startedAt, finishedAt)
 }

swarm/network/stream/testing/testing.go

-// Copyright 2018 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package testing
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/p2p"
-	"github.com/ethereum/go-ethereum/p2p/discover"
-	"github.com/ethereum/go-ethereum/p2p/simulations"
-	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
-	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/swarm/log"
-	"github.com/ethereum/go-ethereum/swarm/network"
-	"github.com/ethereum/go-ethereum/swarm/storage"
-)
-
-type Simulation struct {
-	Net    *simulations.Network
-	Stores []storage.ChunkStore
-	Addrs  []network.Addr
-	IDs    []discover.NodeID
-}
-
-func SetStores(addrs ...network.Addr) ([]storage.ChunkStore, func(), error) {
-	var datadirs []string
-	stores := make([]storage.ChunkStore, len(addrs))
-	var err error
-	for i, addr := range addrs {
-		var datadir string
-		datadir, err = ioutil.TempDir("", "streamer")
-		if err != nil {
-			break
-		}
-		var store storage.ChunkStore
-		params := storage.NewDefaultLocalStoreParams()
-		params.Init(datadir)
-		params.BaseKey = addr.Over()
-		store, err = storage.NewTestLocalStoreForAddr(params)
-		if err != nil {
-			break
-		}
-		datadirs = append(datadirs, datadir)
-		stores[i] = store
-	}
-	teardown := func() {
-		for i, datadir := range datadirs {
-			stores[i].Close()
-			os.RemoveAll(datadir)
-		}
-	}
-	return stores, teardown, err
-}
-
-func NewAdapter(adapterType string, services adapters.Services) (adapter adapters.NodeAdapter, teardown func(), err error) {
-	teardown = func() {}
-	switch adapterType {
-	case "sim":
-		adapter = adapters.NewSimAdapter(services)
-	case "exec":
-		baseDir, err0 := ioutil.TempDir("", "swarm-test")
-		if err0 != nil {
-			return nil, teardown, err0
-		}
-		teardown = func() { os.RemoveAll(baseDir) }
-		adapter = adapters.NewExecAdapter(baseDir)
-	case "docker":
-		adapter, err = adapters.NewDockerAdapter()
-		if err != nil {
-			return nil, teardown, err
-		}
-	default:
-		return nil, teardown, errors.New("adapter needs to be one of sim, exec, docker")
-	}
-	return adapter, teardown, nil
-}
-
-func CheckResult(t *testing.T, result *simulations.StepResult, startedAt, finishedAt time.Time) {
-	t.Logf("Simulation passed in %s", result.FinishedAt.Sub(result.StartedAt))
-	if len(result.Passes) > 1 {
-		var min, max time.Duration
-		var sum int
-		for _, pass := range result.Passes {
-			duration := pass.Sub(result.StartedAt)
-			if sum == 0 || duration < min {
-				min = duration
-			}
-			if duration > max {
-				max = duration
-			}
-			sum += int(duration.Nanoseconds())
-		}
-		t.Logf("Min: %s, Max: %s, Average: %s", min, max, time.Duration(sum/len(result.Passes))*time.Nanosecond)
-	}
-	t.Logf("Setup: %s, Shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt))
-}
-
-type RunConfig struct {
-	Adapter         string
-	Step            *simulations.Step
-	NodeCount       int
-	ConnLevel       int
-	ToAddr          func(discover.NodeID) *network.BzzAddr
-	Services        adapters.Services
-	DefaultService  string
-	EnableMsgEvents bool
-}
-
-func NewSimulation(conf *RunConfig) (*Simulation, func(), error) {
-	// create network
-	nodes := conf.NodeCount
-	adapter, adapterTeardown, err := NewAdapter(conf.Adapter, conf.Services)
-	if err != nil {
-		return nil, adapterTeardown, err
-	}
-	defaultService := "streamer"
-	if conf.DefaultService != "" {
-		defaultService = conf.DefaultService
-	}
-	net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{
-		ID:             "0",
-		DefaultService: defaultService,
-	})
-	teardown := func() {
-		adapterTeardown()
-		net.Shutdown()
-	}
-	ids := make([]discover.NodeID, nodes)
-	addrs := make([]network.Addr, nodes)
-	// start nodes
-	for i := 0; i < nodes; i++ {
-		nodeconf := adapters.RandomNodeConfig()
-		nodeconf.EnableMsgEvents = conf.EnableMsgEvents
-		node, err := net.NewNodeWithConfig(nodeconf)
-		if err != nil {
-			return nil, teardown, fmt.Errorf("error creating node: %s", err)
-		}
-		ids[i] = node.ID()
-		addrs[i] = conf.ToAddr(ids[i])
-	}
-	// set nodes number of Stores available
-	stores, storeTeardown, err := SetStores(addrs...)
-	teardown = func() {
-		net.Shutdown()
-		adapterTeardown()
-		storeTeardown()
-	}
-	if err != nil {
-		return nil, teardown, err
-	}
-	s := &Simulation{
-		Net:    net,
-		Stores: stores,
-		IDs:    ids,
-		Addrs:  addrs,
-	}
-	return s, teardown, nil
-}
-
-func (s *Simulation) Run(ctx context.Context, conf *RunConfig) (*simulations.StepResult, error) {
-	// bring up nodes, launch the servive
-	nodes := conf.NodeCount
-	conns := conf.ConnLevel
-	for i := 0; i < nodes; i++ {
-		if err := s.Net.Start(s.IDs[i]); err != nil {
-			return nil, fmt.Errorf("error starting node %s: %s", s.IDs[i].TerminalString(), err)
-		}
-	}
-	// run a simulation which connects the 10 nodes in a chain
-	wg := sync.WaitGroup{}
-	for i := range s.IDs {
-		// collect the overlay addresses, to
-		for j := 0; j < conns; j++ {
-			var k int
-			if j == 0 {
-				k = i - 1
-			} else {
-				k = rand.Intn(len(s.IDs))
-			}
-			if i > 0 {
-				wg.Add(1)
-				go func(i, k int) {
-					defer wg.Done()
-					s.Net.Connect(s.IDs[i], s.IDs[k])
-				}(i, k)
-			}
-		}
-	}
-	wg.Wait()
-	log.Info(fmt.Sprintf("simulation with %v nodes", len(s.Addrs)))
-
-	// create an only locally retrieving FileStore for the pivot node to test
-	// if retriee requests have arrived
-	result := simulations.NewSimulation(s.Net).Run(ctx, conf.Step)
-	return result, nil
-}
-
-// WatchDisconnections subscribes to admin peerEvents and sends peer event drop
-// errors to the errc channel. Channel quitC signals the termination of the event loop.
-// Returned doneC will be closed after the rpc subscription is unsubscribed,
-// signaling that simulations network is safe to shutdown.
-func WatchDisconnections(id discover.NodeID, client *rpc.Client, errc chan error, quitC chan struct{}) (doneC <-chan struct{}, err error) {
-	events := make(chan *p2p.PeerEvent)
-	sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents")
-	if err != nil {
-		return nil, fmt.Errorf("error getting peer events for node %v: %s", id, err)
-	}
-	c := make(chan struct{})
-	go func() {
-		defer func() {
-			log.Trace("watch disconnections: unsubscribe", "id", id)
-			sub.Unsubscribe()
-			close(c)
-		}()
-		for {
-			select {
-			case <-quitC:
-				return
-			case e := <-events:
-				if e.Type == p2p.PeerEventTypeDrop {
-					select {
-					case errc <- fmt.Errorf("peerEvent for node %v: %v", id, e):
-					case <-quitC:
-						return
-					}
-				}
-			case err := <-sub.Err():
-				if err != nil {
-					select {
-					case errc <- fmt.Errorf("error getting peer events for node %v: %v", id, err):
-					case <-quitC:
-						return
-					}
-				}
-			}
-		}
-	}()
-	return c, nil
-}
-
-func Trigger(d time.Duration, quitC chan struct{}, ids ...discover.NodeID) chan discover.NodeID {
-	trigger := make(chan discover.NodeID)
-	go func() {
-		defer close(trigger)
-		ticker := time.NewTicker(d)
-		defer ticker.Stop()
-		// we are only testing the pivot node (net.Nodes[0])
-		for range ticker.C {
-			for _, id := range ids {
-				select {
-				case trigger <- id:
-				case <-quitC:
-					return
-				}
-			}
-		}
-	}()
-	return trigger
-}
-
-func (sim *Simulation) CallClient(id discover.NodeID, f func(*rpc.Client) error) error {
-	node := sim.Net.GetNode(id)
-	if node == nil {
-		return fmt.Errorf("unknown node: %s", id)
-	}
-	client, err := node.Client()
-	if err != nil {
-		return fmt.Errorf("error getting node client: %s", err)
-	}
-	return f(client)
-}