cmd/swarm, swarm/api/client: add HTTP API client and 'swarm ls' command (#3742)

This adds a swarm ls command which lists files and directories stored in a
manifest. Rather than listing all files, it uses "directory prefixes" in case there are a
lot of files in a manifest but you just want to traverse it.

This also includes some refactoring to the tests and the introduction of a
swarm/api/client package to make things easier to test.

cmd/swarm/list.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
+	"gopkg.in/urfave/cli.v1"
+)
+
+func list(ctx *cli.Context) {
+	args := ctx.Args()
+
+	if len(args) < 1 {
+		utils.Fatalf("Please supply a manifest reference as the first argument")
+	} else if len(args) > 2 {
+		utils.Fatalf("Too many arguments - usage 'swarm ls manifest [prefix]'")
+	}
+	manifest := args[0]
+
+	var prefix string
+	if len(args) == 2 {
+		prefix = args[1]
+	}
+
+	bzzapi := strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
+	client := swarm.NewClient(bzzapi)
+	entries, err := client.ManifestFileList(manifest, prefix)
+	if err != nil {
+		utils.Fatalf("Failed to generate file and directory list: %s", err)
+	}
+
+	w := tabwriter.NewWriter(os.Stdout, 1, 2, 2, ' ', 0)
+	defer w.Flush()
+	fmt.Fprintln(w, "HASH\tCONTENT TYPE\tPATH")
+	for _, entry := range entries {
+		fmt.Fprintf(w, "%s\t%s\t%s\n", entry.Hash, entry.ContentType, entry.Path)
+	}
+}

cmd/swarm/main.go

@@ -145,6 +145,15 @@ The output of this command is supposed to be machine-readable.
 			ArgsUsage: " <file>",
 			Description: `
 "upload a file or directory to swarm using the HTTP API and prints the root hash",
+`,
+		},
+		{
+			Action:    list,
+			Name:      "ls",
+			Usage:     "list files and directories contained in a manifest",
+			ArgsUsage: " <manifest> [<prefix>]",
+			Description: `
+Lists files and directories contained in a manifest.
 `,
 		},
 		{

cmd/swarm/manifest.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -41,7 +42,7 @@ func add(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        swarm.Manifest
 	)
 
 	if len(args) > 3 {
@@ -75,7 +76,7 @@ func update(ctx *cli.Context) {
 
 		ctype        string
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        swarm.Manifest
 	)
 	if len(args) > 3 {
 		ctype = args[3]
@@ -105,7 +106,7 @@ func remove(ctx *cli.Context) {
 		path  = args[1]
 
 		wantManifest = ctx.GlobalBoolT(SwarmWantManifestFlag.Name)
-		mroot        manifest
+		mroot        swarm.Manifest
 	)
 
 	newManifest := removeEntryFromManifest(ctx, mhash, path)
@@ -123,21 +124,21 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 
 	var (
 		bzzapi           = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client           = &client{api: bzzapi}
-		longestPathEntry = manifestEntry{
+		client           = swarm.NewClient(bzzapi)
+		longestPathEntry = swarm.ManifestEntry{
 			Path:        "",
 			Hash:        "",
 			ContentType: "",
 		}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
 
 	//TODO: check if the "hash" to add is valid and present in swarm
-	_, err = client.downloadManifest(hash)
+	_, err = client.DownloadManifest(hash)
 	if err != nil {
 		utils.Fatalf("Hash to add is not present: %v", err)
 	}
@@ -162,7 +163,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		newHash := addEntryToManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := swarm.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -172,7 +173,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		mroot = newMRoot
 	} else {
 		// Add the entry in the leaf Manifest
-		newEntry := manifestEntry{
+		newEntry := swarm.ManifestEntry{
 			Path:        path,
 			Hash:        hash,
 			ContentType: ctype,
@@ -180,7 +181,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		mroot.Entries = append(mroot.Entries, newEntry)
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -192,20 +193,20 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 
 	var (
 		bzzapi   = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client   = &client{api: bzzapi}
-		newEntry = manifestEntry{
+		client   = swarm.NewClient(bzzapi)
+		newEntry = swarm.ManifestEntry{
 			Path:        "",
 			Hash:        "",
 			ContentType: "",
 		}
-		longestPathEntry = manifestEntry{
+		longestPathEntry = swarm.ManifestEntry{
 			Path:        "",
 			Hash:        "",
 			ContentType: "",
 		}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
@@ -236,7 +237,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		newHash := updateEntryInManifest(ctx, longestPathEntry.Hash, newPath, hash, ctype)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := swarm.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -249,10 +250,10 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 
 	if newEntry.Path != "" {
 		// Replace the hash for leaf Manifest
-		newMRoot := manifest{}
+		newMRoot := swarm.Manifest{}
 		for _, entry := range mroot.Entries {
 			if newEntry.Path == entry.Path {
-				myEntry := manifestEntry{
+				myEntry := swarm.ManifestEntry{
 					Path:        entry.Path,
 					Hash:        hash,
 					ContentType: ctype,
@@ -265,7 +266,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		mroot = newMRoot
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -276,20 +277,20 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	var (
 		bzzapi        = strings.TrimRight(ctx.GlobalString(SwarmApiFlag.Name), "/")
-		client        = &client{api: bzzapi}
-		entryToRemove = manifestEntry{
+		client        = swarm.NewClient(bzzapi)
+		entryToRemove = swarm.ManifestEntry{
 			Path:        "",
 			Hash:        "",
 			ContentType: "",
 		}
-		longestPathEntry = manifestEntry{
+		longestPathEntry = swarm.ManifestEntry{
 			Path:        "",
 			Hash:        "",
 			ContentType: "",
 		}
 	)
 
-	mroot, err := client.downloadManifest(mhash)
+	mroot, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
@@ -318,7 +319,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		newHash := removeEntryFromManifest(ctx, longestPathEntry.Hash, newPath)
 
 		// Replace the hash for parent Manifests
-		newMRoot := manifest{}
+		newMRoot := swarm.Manifest{}
 		for _, entry := range mroot.Entries {
 			if longestPathEntry.Path == entry.Path {
 				entry.Hash = newHash
@@ -330,7 +331,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 
 	if entryToRemove.Path != "" {
 		// remove the entry in this Manifest
-		newMRoot := manifest{}
+		newMRoot := swarm.Manifest{}
 		for _, entry := range mroot.Entries {
 			if entryToRemove.Path != entry.Path {
 				newMRoot.Entries = append(newMRoot.Entries, entry)
@@ -339,7 +340,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		mroot = newMRoot
 	}
 
-	newManifestHash, err := client.uploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}

cmd/swarm/upload.go

@@ -18,21 +18,15 @@
 package main
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
-	"io/ioutil"
-	"mime"
-	"net/http"
 	"os"
 	"os/user"
 	"path"
-	"path/filepath"
 	"strings"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/log"
+	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
 	"gopkg.in/urfave/cli.v1"
 )
 
@@ -50,7 +44,7 @@ func upload(ctx *cli.Context) {
 
 	var (
 		file   = args[0]
-		client = &client{api: bzzapi}
+		client = swarm.NewClient(bzzapi)
 	)
 	fi, err := os.Stat(expandPath(file))
 	if err != nil {
@@ -63,25 +57,25 @@ func upload(ctx *cli.Context) {
 		if !wantManifest {
 			utils.Fatalf("Manifest is required for directory uploads")
 		}
-		mhash, err := client.uploadDirectory(file, defaultPath)
+		mhash, err := client.UploadDirectory(file, defaultPath)
 		if err != nil {
 			utils.Fatalf("Failed to upload directory: %v", err)
 		}
 		fmt.Println(mhash)
 		return
 	}
-	entry, err := client.uploadFile(file, fi)
+	entry, err := client.UploadFile(file, fi)
 	if err != nil {
 		utils.Fatalf("Upload failed: %v", err)
 	}
-	mroot := manifest{[]manifestEntry{entry}}
+	mroot := swarm.Manifest{Entries: []swarm.ManifestEntry{entry}}
 	if !wantManifest {
 		// Print the manifest. This is the only output to stdout.
 		mrootJSON, _ := json.MarshalIndent(mroot, "", "  ")
 		fmt.Println(string(mrootJSON))
 		return
 	}
-	hash, err := client.uploadManifest(mroot)
+	hash, err := client.UploadManifest(mroot)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -111,148 +105,3 @@ func homeDir() string {
 	}
 	return ""
 }
-
-// client wraps interaction with the swarm HTTP gateway.
-type client struct {
-	api string
-}
-
-// manifest is the JSON representation of a swarm manifest.
-type manifestEntry struct {
-	Hash        string `json:"hash,omitempty"`
-	ContentType string `json:"contentType,omitempty"`
-	Path        string `json:"path,omitempty"`
-}
-
-// manifest is the JSON representation of a swarm manifest.
-type manifest struct {
-	Entries []manifestEntry `json:"entries,omitempty"`
-}
-
-func (c *client) uploadDirectory(dir string, defaultPath string) (string, error) {
-	mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
-	if err != nil {
-		return "", fmt.Errorf("failed to upload empty manifest")
-	}
-	if len(defaultPath) > 0 {
-		fi, err := os.Stat(defaultPath)
-		if err != nil {
-			return "", err
-		}
-		mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
-		if err != nil {
-			return "", err
-		}
-	}
-	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
-	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
-		if err != nil || fi.IsDir() {
-			return err
-		}
-		if !strings.HasPrefix(path, dir) {
-			return fmt.Errorf("path %s outside directory %s", path, dir)
-		}
-		uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
-		mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
-		return err
-	})
-	return mhash, err
-}
-
-func (c *client) uploadFile(file string, fi os.FileInfo) (manifestEntry, error) {
-	hash, err := c.uploadFileContent(file, fi)
-	m := manifestEntry{
-		Hash:        hash,
-		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
-	}
-	return m, err
-}
-
-func (c *client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(file)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
-	return c.postRaw("application/octet-stream", fi.Size(), fd)
-}
-
-func (c *client) uploadManifest(m manifest) (string, error) {
-	jsm, err := json.Marshal(m)
-	if err != nil {
-		panic(err)
-	}
-	log.Info("Uploading swarm manifest")
-	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
-}
-
-func (c *client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
-	fd, err := os.Open(fpath)
-	if err != nil {
-		return "", err
-	}
-	defer fd.Close()
-	log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
-	req, err := http.NewRequest("PUT", c.api+"/bzz:/"+mhash+"/"+path, fd)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
-	req.ContentLength = fi.Size()
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
-}
-
-func (c *client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
-	req, err := http.NewRequest("POST", c.api+"/bzzr:/", body)
-	if err != nil {
-		return "", err
-	}
-	req.Header.Set("content-type", mimetype)
-	req.ContentLength = size
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode >= 400 {
-		return "", fmt.Errorf("bad status: %s", resp.Status)
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-	return string(content), err
-}
-
-func (c *client) downloadManifest(mhash string) (manifest, error) {
-
-	mroot := manifest{}
-	req, err := http.NewRequest("GET", c.api+"/bzzr:/"+mhash, nil)
-	if err != nil {
-		return mroot, err
-	}
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return mroot, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode >= 400 {
-		return mroot, fmt.Errorf("bad status: %s", resp.Status)
-
-	}
-	content, err := ioutil.ReadAll(resp.Body)
-
-	err = json.Unmarshal(content, &mroot)
-	if err != nil {
-		return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
-	}
-	return mroot, err
-}

swarm/api/client/client.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+var (
+	DefaultGateway = "http://localhost:8500"
+	DefaultClient  = NewClient(DefaultGateway)
+)
+
+// Manifest represents a swarm manifest.
+type Manifest struct {
+	Entries []ManifestEntry `json:"entries,omitempty"`
+}
+
+// ManifestEntry represents an entry in a swarm manifest.
+type ManifestEntry struct {
+	Hash        string `json:"hash,omitempty"`
+	ContentType string `json:"contentType,omitempty"`
+	Path        string `json:"path,omitempty"`
+}
+
+func NewClient(gateway string) *Client {
+	return &Client{
+		Gateway: gateway,
+	}
+}
+
+// Client wraps interaction with a swarm HTTP gateway.
+type Client struct {
+	Gateway string
+}
+
+func (c *Client) UploadDirectory(dir string, defaultPath string) (string, error) {
+	mhash, err := c.postRaw("application/json", 2, ioutil.NopCloser(bytes.NewReader([]byte("{}"))))
+	if err != nil {
+		return "", fmt.Errorf("failed to upload empty manifest")
+	}
+	if len(defaultPath) > 0 {
+		fi, err := os.Stat(defaultPath)
+		if err != nil {
+			return "", err
+		}
+		mhash, err = c.uploadToManifest(mhash, "", defaultPath, fi)
+		if err != nil {
+			return "", err
+		}
+	}
+	prefix := filepath.ToSlash(filepath.Clean(dir)) + "/"
+	err = filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
+		if err != nil || fi.IsDir() {
+			return err
+		}
+		if !strings.HasPrefix(path, dir) {
+			return fmt.Errorf("path %s outside directory %s", path, dir)
+		}
+		uripath := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), prefix)
+		mhash, err = c.uploadToManifest(mhash, uripath, path, fi)
+		return err
+	})
+	return mhash, err
+}
+
+func (c *Client) UploadFile(file string, fi os.FileInfo) (ManifestEntry, error) {
+	hash, err := c.uploadFileContent(file, fi)
+	m := ManifestEntry{
+		Hash:        hash,
+		ContentType: mime.TypeByExtension(filepath.Ext(fi.Name())),
+	}
+	return m, err
+}
+
+func (c *Client) uploadFileContent(file string, fi os.FileInfo) (string, error) {
+	fd, err := os.Open(file)
+	if err != nil {
+		return "", err
+	}
+	defer fd.Close()
+	log.Info("Uploading swarm content", "file", file, "bytes", fi.Size())
+	return c.postRaw("application/octet-stream", fi.Size(), fd)
+}
+
+func (c *Client) UploadManifest(m Manifest) (string, error) {
+	jsm, err := json.Marshal(m)
+	if err != nil {
+		panic(err)
+	}
+	log.Info("Uploading swarm manifest")
+	return c.postRaw("application/json", int64(len(jsm)), ioutil.NopCloser(bytes.NewReader(jsm)))
+}
+
+func (c *Client) uploadToManifest(mhash string, path string, fpath string, fi os.FileInfo) (string, error) {
+	fd, err := os.Open(fpath)
+	if err != nil {
+		return "", err
+	}
+	defer fd.Close()
+	log.Info("Uploading swarm content and path", "file", fpath, "bytes", fi.Size(), "path", path)
+	req, err := http.NewRequest("PUT", c.Gateway+"/bzz:/"+mhash+"/"+path, fd)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("content-type", mime.TypeByExtension(filepath.Ext(fi.Name())))
+	req.ContentLength = fi.Size()
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 400 {
+		return "", fmt.Errorf("bad status: %s", resp.Status)
+	}
+	content, err := ioutil.ReadAll(resp.Body)
+	return string(content), err
+}
+
+func (c *Client) postRaw(mimetype string, size int64, body io.ReadCloser) (string, error) {
+	req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", body)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("content-type", mimetype)
+	req.ContentLength = size
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode >= 400 {
+		return "", fmt.Errorf("bad status: %s", resp.Status)
+	}
+	content, err := ioutil.ReadAll(resp.Body)
+	return string(content), err
+}
+
+func (c *Client) DownloadManifest(mhash string) (Manifest, error) {
+
+	mroot := Manifest{}
+	req, err := http.NewRequest("GET", c.Gateway+"/bzzr:/"+mhash, nil)
+	if err != nil {
+		return mroot, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return mroot, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode >= 400 {
+		return mroot, fmt.Errorf("bad status: %s", resp.Status)
+
+	}
+	content, err := ioutil.ReadAll(resp.Body)
+
+	err = json.Unmarshal(content, &mroot)
+	if err != nil {
+		return mroot, fmt.Errorf("Manifest %v is malformed: %v", mhash, err)
+	}
+	return mroot, err
+}
+
+// ManifestFileList downloads the manifest with the given hash and generates a
+// list of files and directory prefixes which have the specified prefix.
+//
+// For example, if the manifest represents the following directory structure:
+//
+// file1.txt
+// file2.txt
+// dir1/file3.txt
+// dir1/dir2/file4.txt
+//
+// Then:
+//
+// - a prefix of ""      would return [dir1/, file1.txt, file2.txt]
+// - a prefix of "file"  would return [file1.txt, file2.txt]
+// - a prefix of "dir1/" would return [dir1/dir2/, dir1/file3.txt]
+func (c *Client) ManifestFileList(hash, prefix string) (entries []ManifestEntry, err error) {
+	manifest, err := c.DownloadManifest(hash)
+	if err != nil {
+		return nil, err
+	}
+
+	// handleFile handles a manifest entry which is a direct reference to a
+	// file (i.e. it is not a swarm manifest)
+	handleFile := func(entry ManifestEntry) {
+		// ignore the file if it doesn't have the specified prefix
+		if !strings.HasPrefix(entry.Path, prefix) {
+			return
+		}
+		// if the path after the prefix contains a directory separator,
+		// add a directory prefix to the entries, otherwise add the
+		// file
+		suffix := strings.TrimPrefix(entry.Path, prefix)
+		if sepIndex := strings.Index(suffix, "/"); sepIndex > -1 {
+			entries = append(entries, ManifestEntry{
+				Path:        prefix + suffix[:sepIndex+1],
+				ContentType: "DIR",
+			})
+		} else {
+			if entry.Path == "" {
+				entry.Path = "/"
+			}
+			entries = append(entries, entry)
+		}
+	}
+
+	// handleManifest handles a manifest entry which is a reference to
+	// another swarm manifest.
+	handleManifest := func(entry ManifestEntry) error {
+		// if the manifest's path is a prefix of the specified prefix
+		// then just recurse into the manifest by stripping its path
+		// from the prefix
+		if strings.HasPrefix(prefix, entry.Path) {
+			subPrefix := strings.TrimPrefix(prefix, entry.Path)
+			subEntries, err := c.ManifestFileList(entry.Hash, subPrefix)
+			if err != nil {
+				return err
+			}
+			// prefix the manifest's path to the sub entries and
+			// add them to the returned entries
+			for i, subEntry := range subEntries {
+				subEntry.Path = entry.Path + subEntry.Path
+				subEntries[i] = subEntry
+			}
+			entries = append(entries, subEntries...)
+			return nil
+		}
+
+		// if the manifest's path has the specified prefix, then if the
+		// path after the prefix contains a directory separator, add a
+		// directory prefix to the entries, otherwise recurse into the
+		// manifest
+		if strings.HasPrefix(entry.Path, prefix) {
+			suffix := strings.TrimPrefix(entry.Path, prefix)
+			sepIndex := strings.Index(suffix, "/")
+			if sepIndex > -1 {
+				entries = append(entries, ManifestEntry{
+					Path:        prefix + suffix[:sepIndex+1],
+					ContentType: "DIR",
+				})
+				return nil
+			}
+			subEntries, err := c.ManifestFileList(entry.Hash, "")
+			if err != nil {
+				return err
+			}
+			// prefix the manifest's path to the sub entries and
+			// add them to the returned entries
+			for i, subEntry := range subEntries {
+				subEntry.Path = entry.Path + subEntry.Path
+				subEntries[i] = subEntry
+			}
+			entries = append(entries, subEntries...)
+			return nil
+		}
+		return nil
+	}
+
+	for _, entry := range manifest.Entries {
+		if entry.ContentType == "application/bzz-manifest+json" {
+			if err := handleManifest(entry); err != nil {
+				return nil, err
+			}
+		} else {
+			handleFile(entry)
+		}
+	}
+
+	return
+}

swarm/api/client/client_test.go

+// Copyright 2016 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package client
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sort"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/testutil"
+)
+
+func TestClientManifestFileList(t *testing.T) {
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
+
+	dir, err := ioutil.TempDir("", "swarm-client-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	files := []string{
+		"file1.txt",
+		"file2.txt",
+		"dir1/file3.txt",
+		"dir1/file4.txt",
+		"dir2/file5.txt",
+		"dir2/dir3/file6.txt",
+		"dir2/dir4/file7.txt",
+		"dir2/dir4/file8.txt",
+	}
+	for _, file := range files {
+		path := filepath.Join(dir, file)
+		if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+			t.Fatalf("error creating dir for %s: %s", path, err)
+		}
+		if err := ioutil.WriteFile(path, []byte("data"), 0644); err != nil {
+			t.Fatalf("error writing file %s: %s", path, err)
+		}
+	}
+
+	client := NewClient(srv.URL)
+
+	hash, err := client.UploadDirectory(dir, "")
+	if err != nil {
+		t.Fatalf("error uploading directory: %s", err)
+	}
+
+	ls := func(prefix string) []string {
+		entries, err := client.ManifestFileList(hash, prefix)
+		if err != nil {
+			t.Fatal(err)
+		}
+		paths := make([]string, len(entries))
+		for i, entry := range entries {
+			paths[i] = entry.Path
+		}
+		sort.Strings(paths)
+		return paths
+	}
+
+	tests := map[string][]string{
+		"":                    []string{"dir1/", "dir2/", "file1.txt", "file2.txt"},
+		"file":                []string{"file1.txt", "file2.txt"},
+		"file1":               []string{"file1.txt"},
+		"file2.txt":           []string{"file2.txt"},
+		"file12":              []string{},
+		"dir":                 []string{"dir1/", "dir2/"},
+		"dir1":                []string{"dir1/"},
+		"dir1/":               []string{"dir1/file3.txt", "dir1/file4.txt"},
+		"dir1/file":           []string{"dir1/file3.txt", "dir1/file4.txt"},
+		"dir1/file3.txt":      []string{"dir1/file3.txt"},
+		"dir1/file34":         []string{},
+		"dir2/":               []string{"dir2/dir3/", "dir2/dir4/", "dir2/file5.txt"},
+		"dir2/file":           []string{"dir2/file5.txt"},
+		"dir2/dir":            []string{"dir2/dir3/", "dir2/dir4/"},
+		"dir2/dir3/":          []string{"dir2/dir3/file6.txt"},
+		"dir2/dir4/":          []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
+		"dir2/dir4/file":      []string{"dir2/dir4/file7.txt", "dir2/dir4/file8.txt"},
+		"dir2/dir4/file7.txt": []string{"dir2/dir4/file7.txt"},
+		"dir2/dir4/file78":    []string{},
+	}
+	for prefix, expected := range tests {
+		actual := ls(prefix)
+		if !reflect.DeepEqual(actual, expected) {
+			t.Fatalf("expected prefix %q to return paths %v, got %v", prefix, expected, actual)
+		}
+	}
+}

swarm/api/http/server.go

@@ -56,9 +56,9 @@ type sequentialReader struct {
 	lock   sync.Mutex
 }
 
-// Server is the basic configuration needs for the HTTP server and also
+// ServerConfig is the basic configuration needed for the HTTP server and also
 // includes CORS settings.
-type Server struct {
+type ServerConfig struct {
 	Addr       string
 	CorsString string
 }
@@ -69,13 +69,9 @@ type Server struct {
 // https://github.com/atom/electron/blob/master/docs/api/protocol.md
 
 // starts up http server
-func StartHttpServer(api *api.Api, server *Server) {
-	serveMux := http.NewServeMux()
-	serveMux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		handler(w, r, api)
-	})
+func StartHttpServer(api *api.Api, config *ServerConfig) {
 	var allowedOrigins []string
-	for _, domain := range strings.Split(server.CorsString, ",") {
+	for _, domain := range strings.Split(config.CorsString, ",") {
 		allowedOrigins = append(allowedOrigins, strings.TrimSpace(domain))
 	}
 	c := cors.New(cors.Options{
@@ -84,13 +80,21 @@ func StartHttpServer(api *api.Api, server *Server) {
 		MaxAge:         600,
 		AllowedHeaders: []string{"*"},
 	})
-	hdlr := c.Handler(serveMux)
+	hdlr := c.Handler(NewServer(api))
+
+	go http.ListenAndServe(config.Addr, hdlr)
+	log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", config.Addr))
+}
 
-	go http.ListenAndServe(server.Addr, hdlr)
-	log.Info(fmt.Sprintf("Swarm HTTP proxy started on localhost:%s", server.Addr))
+func NewServer(api *api.Api) *Server {
+	return &Server{api}
+}
+
+type Server struct {
+	api *api.Api
 }
 
-func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
+func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	requestURL := r.URL
 	// This is wrong
 	//	if requestURL.Host == "" {
@@ -135,7 +139,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 			http.Error(w, "Missing Content-Length header in request.", http.StatusBadRequest)
 			return
 		}
-		key, err := a.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
+		key, err := s.api.Store(io.LimitReader(r.Body, r.ContentLength), r.ContentLength, nil)
 		if err == nil {
 			log.Debug(fmt.Sprintf("Content for %v stored", key.Log()))
 		} else {
@@ -160,7 +164,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 				mime := r.Header.Get("Content-Type")
 				// TODO proper root hash separation
 				log.Debug(fmt.Sprintf("Modify '%s' to store %v as '%s'.", path, key.Log(), mime))
-				newKey, err := a.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
+				newKey, err := s.api.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
 				if err == nil {
 					log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
 					w.Header().Set("Content-Type", "text/plain")
@@ -178,7 +182,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 		} else {
 			path = api.RegularSlashes(path)
 			log.Debug(fmt.Sprintf("Delete '%s'.", path))
-			newKey, err := a.Modify(path, "", "", nameresolver)
+			newKey, err := s.api.Modify(path, "", "", nameresolver)
 			if err == nil {
 				log.Debug(fmt.Sprintf("Swarm replaced manifest by '%s'", newKey))
 				w.Header().Set("Content-Type", "text/plain")
@@ -199,16 +203,16 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 			parsedurl, _ := api.Parse(path)
 
 			if parsedurl == path {
-				key, err := a.Resolve(parsedurl, nameresolver)
+				key, err := s.api.Resolve(parsedurl, nameresolver)
 				if err != nil {
 					log.Error(fmt.Sprintf("%v", err))
 					http.Error(w, err.Error(), http.StatusBadRequest)
 					return
 				}
-				reader = a.Retrieve(key)
+				reader = s.api.Retrieve(key)
 			} else {
 				var status int
-				readertmp, _, status, err := a.Get(path, nameresolver)
+				readertmp, _, status, err := s.api.Get(path, nameresolver)
 				if err != nil {
 					http.Error(w, err.Error(), status)
 					return
@@ -247,7 +251,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 				http.Redirect(w, r, path+"/", http.StatusFound)
 				return
 			}
-			reader, mimeType, status, err := a.Get(path, nameresolver)
+			reader, mimeType, status, err := s.api.Get(path, nameresolver)
 			if err != nil {
 				if _, ok := err.(api.ErrResolve); ok {
 					log.Debug(fmt.Sprintf("%v", err))

swarm/api/http/server_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-package http
+package http_test
 
 import (
 	"bytes"
@@ -22,19 +22,16 @@ import (
 	"net/http"
 	"sync"
 	"testing"
-	"time"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/testutil"
 )
 
 func TestBzzrGetPath(t *testing.T) {
 
 	var err error
 
-	maxproxyattempts := 3
-
 	testmanifest := []string{
 		`{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`,
 		`{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`,
@@ -54,61 +51,30 @@ func TestBzzrGetPath(t *testing.T) {
 
 	key := [3]storage.Key{}
 
-	dir, _ := ioutil.TempDir("", "bzz-storage-test")
-
-	storeparams := &storage.StoreParams{
-		ChunkDbPath:   dir,
-		DbCapacity:    5000000,
-		CacheCapacity: 5000,
-		Radius:        0,
-	}
-
-	localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams)
-	if err != nil {
-		t.Fatal(err)
-	}
-	chunker := storage.NewTreeChunker(storage.NewChunkerParams())
-	dpa := &storage.DPA{
-		Chunker:    chunker,
-		ChunkStore: localStore,
-	}
-	dpa.Start()
-	defer dpa.Stop()
+	srv := testutil.NewTestSwarmServer(t)
+	defer srv.Close()
 
 	wg := &sync.WaitGroup{}
 
 	for i, mf := range testmanifest {
 		reader[i] = bytes.NewReader([]byte(mf))
-		key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil)
+		key[i], err = srv.Dpa.Store(reader[i], int64(len(mf)), wg, nil)
 		if err != nil {
 			t.Fatal(err)
 		}
 		wg.Wait()
 	}
 
-	a := api.NewApi(dpa, nil)
-
-	/// \todo iterate port numbers up if fail
-	StartHttpServer(a, &Server{Addr: "127.0.0.1:8504", CorsString: ""})
-	// how to wait for ListenAndServe to have initialized? This is pretty cruuuude
-	// if we fix it we don't need maxproxyattempts anymore either
-	time.Sleep(1000 * time.Millisecond)
-	for i := 0; i <= maxproxyattempts; i++ {
-		_, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
-		if i == maxproxyattempts {
-			t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err)
-		} else if err != nil {
-			time.Sleep(100 * time.Millisecond)
-			continue
-		}
-		break
+	_, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
+	if err != nil {
+		t.Fatalf("Failed to connect to proxy: %v", err)
 	}
 
 	for k, v := range testrequests {
 		var resp *http.Response
 		var respbody []byte
 
-		url := "http://127.0.0.1:8504/bzzr:/"
+		url := srv.URL + "/bzzr:/"
 		if k[:] != "" {
 			url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
 		}

swarm/swarm.go

@@ -195,7 +195,10 @@ func (self *Swarm) Start(net *p2p.Server) error {
 	// start swarm http proxy server
 	if self.config.Port != "" {
 		addr := ":" + self.config.Port
-		go httpapi.StartHttpServer(self.api, &httpapi.Server{Addr: addr, CorsString: self.corsString})
+		go httpapi.StartHttpServer(self.api, &httpapi.ServerConfig{
+			Addr:       addr,
+			CorsString: self.corsString,
+		})
 	}
 
 	log.Debug(fmt.Sprintf("Swarm http proxy started on port: %v", self.config.Port))

swarm/testutil/http.go

+package testutil
+
+import (
+	"io/ioutil"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/api"
+	httpapi "github.com/ethereum/go-ethereum/swarm/api/http"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
+	dir, err := ioutil.TempDir("", "swarm-storage-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	storeparams := &storage.StoreParams{
+		ChunkDbPath:   dir,
+		DbCapacity:    5000000,
+		CacheCapacity: 5000,
+		Radius:        0,
+	}
+	localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams)
+	if err != nil {
+		os.RemoveAll(dir)
+		t.Fatal(err)
+	}
+	chunker := storage.NewTreeChunker(storage.NewChunkerParams())
+	dpa := &storage.DPA{
+		Chunker:    chunker,
+		ChunkStore: localStore,
+	}
+	dpa.Start()
+	a := api.NewApi(dpa, nil)
+	srv := httptest.NewServer(httpapi.NewServer(a))
+	return &TestSwarmServer{
+		Server: srv,
+		Dpa:    dpa,
+		dir:    dir,
+	}
+}
+
+type TestSwarmServer struct {
+	*httptest.Server
+
+	Dpa *storage.DPA
+	dir string
+}
+
+func (t *TestSwarmServer) Close() {
+	t.Server.Close()
+	t.Dpa.Stop()
+	os.RemoveAll(t.dir)
+}