cmd/swarm, swarm/api: bzzr improve + networkid prio

fixes #3444
fixes #3494
networkid override

Added comments to explain why test against 0 appears twice

* Command line overrides saved config, saved config overrides system default

---

fixes #3476
bzzr get with path

Finally a hopefully clean commit for this PR
Added check for empty path to avoid SIGSEGV in path parser and resolver
Added requested tests for empty path and non-existing manifest.
However signature for StartHTTPServer had changed.
Now it's hacked as so:

	StartHttpServer(api.API, &Server{Addr: "127.0.0.1:8504", CorsString: ""})

* Parse url before resolve when path and ENS is supplied, example
* swarm/api/http proxy server test for retrieval of subpath through get
* Removed nil entry assignment on subtrie leaf in recursive key retrieval
* Cleaned up path-or-no-path condition in proxy server get handler
* swarm: processed with gofmt refers to lash/go-ethereum@90daa7a
* swarm: Added public access method Parse alias to parse
* swarm: processed with gofmt References nolash/go-ethereum@2ec3fd7
* Rename parse to Parse, removed alias

cmd/swarm/main.go

@@ -39,7 +39,6 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/swarm"
 	bzzapi "github.com/ethereum/go-ethereum/swarm/api"
-	"github.com/ethereum/go-ethereum/swarm/network"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -76,7 +75,6 @@ var (
 	SwarmNetworkIdFlag = cli.IntFlag{
 		Name:  "bzznetworkid",
 		Usage: "Network identifier (integer, default 3=swarm testnet)",
-		Value: network.NetworkId,
 	}
 	SwarmConfigPathFlag = cli.StringFlag{
 		Name:  "bzzconfig",
@@ -242,6 +240,7 @@ func bzzd(ctx *cli.Context) error {
 }
 
 func registerBzzService(ctx *cli.Context, stack *node.Node) {
+
 	prvkey := getAccount(ctx, stack)
 
 	chbookaddr := common.HexToAddress(ctx.GlobalString(ChequebookAddrFlag.Name))
@@ -249,6 +248,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 	if bzzdir == "" {
 		bzzdir = stack.InstanceDir()
 	}
+
 	bzzconfig, err := bzzapi.NewConfig(bzzdir, chbookaddr, prvkey, ctx.GlobalUint64(SwarmNetworkIdFlag.Name))
 	if err != nil {
 		utils.Fatalf("unable to configure swarm: %v", err)
@@ -280,6 +280,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) {
 
 func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey {
 	keyid := ctx.GlobalString(SwarmAccountFlag.Name)
+
 	if keyid == "" {
 		utils.Fatalf("Option %q is required", SwarmAccountFlag.Name)
 	}

swarm/api/api.go

@@ -19,6 +19,7 @@ package api
 import (
 	"fmt"
 	"io"
+	"net/http"
 	"regexp"
 	"strings"
 	"sync"
@@ -71,6 +72,7 @@ type ErrResolve error
 
 // DNS Resolver
 func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) {
+	glog.V(logger.Detail).Infof("Resolving : %v", hostPort)
 	if hashMatcher.MatchString(hostPort) || self.dns == nil {
 		glog.V(logger.Detail).Infof("host is a contentHash: '%v'", hostPort)
 		return storage.Key(common.Hex2Bytes(hostPort)), nil
@@ -86,8 +88,10 @@ func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error
 	glog.V(logger.Detail).Infof("host lookup: %v -> %v", err)
 	return contentHash[:], err
 }
-
-func parse(uri string) (hostPort, path string) {
+func Parse(uri string) (hostPort, path string) {
+	if uri == "" {
+		return
+	}
 	parts := slashes.Split(uri, 3)
 	var i int
 	if len(parts) == 0 {
@@ -111,7 +115,7 @@ func parse(uri string) (hostPort, path string) {
 }
 
 func (self *Api) parseAndResolve(uri string, nameresolver bool) (key storage.Key, hostPort, path string, err error) {
-	hostPort, path = parse(uri)
+	hostPort, path = Parse(uri)
 	//resolving host and port
 	contentHash, err := self.Resolve(hostPort, nameresolver)
 	glog.V(logger.Debug).Infof("Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path)
@@ -153,7 +157,9 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR
 	}
 
 	glog.V(logger.Detail).Infof("getEntry(%s)", path)
+
 	entry, _ := trie.getEntry(path)
+
 	if entry != nil {
 		key = common.Hex2Bytes(entry.Hash)
 		status = entry.Status
@@ -161,6 +167,7 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR
 		glog.V(logger.Detail).Infof("content lookup key: '%v' (%v)", key, mimeType)
 		reader = self.dpa.Retrieve(key)
 	} else {
+		status = http.StatusNotFound
 		err = fmt.Errorf("manifest entry for '%s' not found", path)
 		glog.V(logger.Warn).Infof("%v", err)
 	}

swarm/api/config.go

@@ -85,10 +85,17 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n
 		NetworkId:     networkId,
 	}
 	data, err = ioutil.ReadFile(confpath)
+
+	// if not set in function param, then set default for swarm network, will be overwritten by config file if present
+	if networkId == 0 {
+		self.NetworkId = network.NetworkId
+	}
+
 	if err != nil {
 		if !os.IsNotExist(err) {
 			return
 		}
+
 		// file does not exist
 		// write out config file
 		err = self.Save()
@@ -97,6 +104,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n
 		}
 		return
 	}
+
 	// file exists, deserialise
 	err = json.Unmarshal(data, self)
 	if err != nil {
@@ -109,6 +117,12 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n
 	if keyhex != self.BzzKey {
 		return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey)
 	}
+
+	// if set in function param, replace id set from config file
+	if networkId != 0 {
+		self.NetworkId = networkId
+	}
+
 	self.Swap.SetKey(prvKey)
 
 	if (self.EnsRoot == common.Address{}) {

swarm/api/http/server.go

@@ -32,6 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/logger"
 	"github.com/ethereum/go-ethereum/logger/glog"
 	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/rs/cors"
 )
 
@@ -194,17 +195,34 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 		}
 	case r.Method == "GET" || r.Method == "HEAD":
 		path = trailingSlashes.ReplaceAllString(path, "")
+		if path == "" {
+			http.Error(w, "Empty path not allowed", http.StatusBadRequest)
+			return
+		}
 		if raw {
-			// resolving host
-			key, err := a.Resolve(path, nameresolver)
+			var reader storage.LazySectionReader
+			parsedurl, _ := api.Parse(path)
+
+			if parsedurl == path {
+				key, err := a.Resolve(parsedurl, nameresolver)
 				if err != nil {
 					glog.V(logger.Error).Infof("%v", err)
 					http.Error(w, err.Error(), http.StatusBadRequest)
 					return
 				}
+				reader = a.Retrieve(key)
+			} else {
+				var status int
+				readertmp, _, status, err := a.Get(path, nameresolver)
+				if err != nil {
+					http.Error(w, err.Error(), status)
+					return
+				}
+				reader = readertmp
+			}
 
 			// retrieving content
-			reader := a.Retrieve(key)
+
 			quitC := make(chan bool)
 			size, err := reader.Size(quitC)
 			if err != nil {

swarm/api/http/server_test.go

+package http
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+func TestBzzrGetPath(t *testing.T) {
+
+	var err error
+
+	maxproxyattempts := 3
+
+	testmanifest := []string{
+		`{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`,
+		`{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`,
+		`{"entries":[{"path":"b","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"c","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0}]}`,
+	}
+
+	testrequests := make(map[string]int)
+	testrequests["/"] = 0
+	testrequests["/a"] = 1
+	testrequests["/a/b"] = 2
+	testrequests["/x"] = 0
+	testrequests[""] = 0
+
+	expectedfailrequests := []string{"", "/x"}
+
+	reader := [3]*bytes.Reader{}
+
+	key := [3]storage.Key{}
+
+	dir, _ := ioutil.TempDir("", "bzz-storage-test")
+
+	storeparams := &storage.StoreParams{
+		ChunkDbPath:   dir,
+		DbCapacity:    5000000,
+		CacheCapacity: 5000,
+		Radius:        0,
+	}
+
+	localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams)
+	if err != nil {
+		t.Fatal(err)
+	}
+	chunker := storage.NewTreeChunker(storage.NewChunkerParams())
+	dpa := &storage.DPA{
+		Chunker:    chunker,
+		ChunkStore: localStore,
+	}
+	dpa.Start()
+	defer dpa.Stop()
+
+	wg := &sync.WaitGroup{}
+
+	for i, mf := range testmanifest {
+		reader[i] = bytes.NewReader([]byte(mf))
+		key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		wg.Wait()
+	}
+
+	a := api.NewApi(dpa, nil)
+
+	/// \todo iterate port numbers up if fail
+	StartHttpServer(a, &Server{Addr: "127.0.0.1:8504", CorsString: ""})
+	// how to wait for ListenAndServe to have initialized? This is pretty cruuuude
+	// if we fix it we don't need maxproxyattempts anymore either
+	time.Sleep(1000 * time.Millisecond)
+	for i := 0; i <= maxproxyattempts; i++ {
+		_, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
+		if i == maxproxyattempts {
+			t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err)
+		} else if err != nil {
+			time.Sleep(100 * time.Millisecond)
+			continue
+		}
+		break
+	}
+
+	for k, v := range testrequests {
+		var resp *http.Response
+		var respbody []byte
+
+		url := "http://127.0.0.1:8504/bzzr:/"
+		if k[:] != "" {
+			url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
+		}
+		resp, err = http.Get(url)
+		defer resp.Body.Close()
+		respbody, err = ioutil.ReadAll(resp.Body)
+
+		if string(respbody) != testmanifest[v] {
+			isexpectedfailrequest := false
+
+			for _, r := range expectedfailrequests {
+				if k[:] == r {
+					isexpectedfailrequest = true
+				}
+			}
+			if isexpectedfailrequest == false {
+				t.Fatalf("Response body does not match, expected: %v, got %v", testmanifest[v], string(respbody))
+			}
+		}
+	}
+
+}

swarm/api/manifest.go

@@ -302,7 +302,8 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man
 	if (len(path) >= epl) && (path[:epl] == entry.Path) {
 		glog.V(logger.Detail).Infof("entry.ContentType = %v", entry.ContentType)
 		if entry.ContentType == manifestType {
-			if self.loadSubTrie(entry, quitC) != nil {
+			err := self.loadSubTrie(entry, quitC)
+			if err != nil {
 				return nil, 0
 			}
 			entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC)
@@ -312,8 +313,6 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man
 		} else {
 			pos = epl
 		}
-	} else {
-		entry = nil
 	}
 	return
 }