cmd/swarm: support exporting, importing chunk db (#14868)

cmd/swarm/cleandb.go → cmd/swarm/db.go

@@ -17,22 +17,100 @@
 package main
 
 import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 	"gopkg.in/urfave/cli.v1"
 )
 
-func cleandb(ctx *cli.Context) {
+func dbExport(ctx *cli.Context) {
+	args := ctx.Args()
+	if len(args) != 2 {
+		utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database) and <file> (path to write the tar archive to, - for stdout)")
+	}
+
+	store, err := openDbStore(args[0])
+	if err != nil {
+		utils.Fatalf("error opening local chunk database: %s", err)
+	}
+	defer store.Close()
+
+	var out io.Writer
+	if args[1] == "-" {
+		out = os.Stdout
+	} else {
+		f, err := os.Create(args[1])
+		if err != nil {
+			utils.Fatalf("error opening output file: %s", err)
+		}
+		defer f.Close()
+		out = f
+	}
+
+	count, err := store.Export(out)
+	if err != nil {
+		utils.Fatalf("error exporting local chunk database: %s", err)
+	}
+
+	log.Info(fmt.Sprintf("successfully exported %d chunks", count))
+}
+
+func dbImport(ctx *cli.Context) {
+	args := ctx.Args()
+	if len(args) != 2 {
+		utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database) and <file> (path to read the tar archive from, - for stdin)")
+	}
+
+	store, err := openDbStore(args[0])
+	if err != nil {
+		utils.Fatalf("error opening local chunk database: %s", err)
+	}
+	defer store.Close()
+
+	var in io.Reader
+	if args[1] == "-" {
+		in = os.Stdin
+	} else {
+		f, err := os.Open(args[1])
+		if err != nil {
+			utils.Fatalf("error opening input file: %s", err)
+		}
+		defer f.Close()
+		in = f
+	}
+
+	count, err := store.Import(in)
+	if err != nil {
+		utils.Fatalf("error importing local chunk database: %s", err)
+	}
+
+	log.Info(fmt.Sprintf("successfully imported %d chunks", count))
+}
+
+func dbClean(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) != 1 {
-		utils.Fatalf("Need path to chunks database as the first and only argument")
+		utils.Fatalf("invalid arguments, please specify <chunkdb> (path to a local chunk database)")
 	}
 
-	chunkDbPath := args[0]
-	hash := storage.MakeHashFunc("SHA3")
-	dbStore, err := storage.NewDbStore(chunkDbPath, hash, 10000000, 0)
+	store, err := openDbStore(args[0])
 	if err != nil {
-		utils.Fatalf("Cannot initialise dbstore: %v", err)
+		utils.Fatalf("error opening local chunk database: %s", err)
+	}
+	defer store.Close()
+
+	store.Cleanup()
+}
+
+func openDbStore(path string) (*storage.DbStore, error) {
+	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
+		return nil, fmt.Errorf("invalid chunkdb path: %s", err)
 	}
-	dbStore.Cleanup()
+	hash := storage.MakeHashFunc("SHA3")
+	return storage.NewDbStore(path, hash, 10000000, 0)
 }

cmd/swarm/main.go

@@ -240,12 +240,65 @@ Removes a path from the manifest
 			},
 		},
 		{
-			Action:    cleandb,
+			Name:      "db",
+			Usage:     "manage the local chunk database",
+			ArgsUsage: "db COMMAND",
+			Description: `
+Manage the local chunk database.
+`,
+			Subcommands: []cli.Command{
+				{
+					Action:    dbExport,
+					Name:      "export",
+					Usage:     "export a local chunk database as a tar archive (use - to send to stdout)",
+					ArgsUsage: "<chunkdb> <file>",
+					Description: `
+Export a local chunk database as a tar archive (use - to send to stdout).
+
+    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
+
+The export may be quite large, consider piping the output through the Unix
+pv(1) tool to get a progress bar:
+
+    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar
+`,
+				},
+				{
+					Action:    dbImport,
+					Name:      "import",
+					Usage:     "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
+					ArgsUsage: "<chunkdb> <file>",
+					Description: `
+Import chunks from a tar archive into a local chunk database (use - to read from stdin).
+
+    swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar
+
+The import may be quite large, consider piping the input through the Unix
+pv(1) tool to get a progress bar:
+
+    pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -
+`,
+				},
+				{
+					Action:    dbClean,
+					Name:      "clean",
+					Usage:     "remove corrupt entries from a local chunk database",
+					ArgsUsage: "<chunkdb>",
+					Description: `
+Remove corrupt entries from a local chunk database.
+`,
+				},
+			},
+		},
+		{
+			Action: func(ctx *cli.Context) {
+				utils.Fatalf("ERROR: 'swarm cleandb' has been removed, please use 'swarm db clean'.")
+			},
 			Name:      "cleandb",
-			Usage:     "Cleans database of corrupted entries",
+			Usage:     "DEPRECATED: use 'swarm db clean'",
 			ArgsUsage: " ",
 			Description: `
-Cleans database of corrupted entries.
+DEPRECATED: use 'swarm db clean'.
 `,
 		},
 	}

swarm/storage/dbstore.go

@@ -23,9 +23,13 @@
 package storage
 
 import (
+	"archive/tar"
 	"bytes"
 	"encoding/binary"
+	"encoding/hex"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -260,6 +264,84 @@ func (s *DbStore) collectGarbage(ratio float32) {
 	s.db.Put(keyGCPos, s.gcPos)
 }
 
+// Export writes all chunks from the store to a tar archive, returning the
+// number of chunks written.
+func (s *DbStore) Export(out io.Writer) (int64, error) {
+	tw := tar.NewWriter(out)
+	defer tw.Close()
+
+	it := s.db.NewIterator()
+	defer it.Release()
+	var count int64
+	for ok := it.Seek([]byte{kpIndex}); ok; ok = it.Next() {
+		key := it.Key()
+		if (key == nil) || (key[0] != kpIndex) {
+			break
+		}
+
+		var index dpaDBIndex
+		decodeIndex(it.Value(), &index)
+
+		data, err := s.db.Get(getDataKey(index.Idx))
+		if err != nil {
+			log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v", key[:], err))
+			continue
+		}
+
+		hdr := &tar.Header{
+			Name: hex.EncodeToString(key[1:]),
+			Mode: 0644,
+			Size: int64(len(data)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return count, err
+		}
+		if _, err := tw.Write(data); err != nil {
+			return count, err
+		}
+		count++
+	}
+
+	return count, nil
+}
+
+// Import reads chunks into the store from a tar archive, returning the number
+// of chunks read.
+func (s *DbStore) Import(in io.Reader) (int64, error) {
+	tr := tar.NewReader(in)
+
+	var count int64
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return count, err
+		}
+
+		if len(hdr.Name) != 64 {
+			log.Warn("ignoring non-chunk file", "name", hdr.Name)
+			continue
+		}
+
+		key, err := hex.DecodeString(hdr.Name)
+		if err != nil {
+			log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err)
+			continue
+		}
+
+		data, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return count, err
+		}
+
+		s.Put(&Chunk{Key: key, SData: data})
+		count++
+	}
+
+	return count, nil
+}
+
 func (s *DbStore) Cleanup() {
 	//Iterates over the database and checks that there are no faulty chunks
 	it := s.db.NewIterator()