dump/load all metadata in new binary format (juicedata#5383)

Signed-off-by: jiefenghuang <[email protected]>

cmd/dump.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/DataDog/zstd"
 	"github.com/juicedata/juicefs/pkg/meta"
+	"github.com/juicedata/juicefs/pkg/utils"
 	"github.com/urfave/cli/v2"
 )
 
@@ -33,10 +34,11 @@ func cmdDump() *cli.Command {
 		Name:      "dump",
 		Action:    dump,
 		Category:  "ADMIN",
-		Usage:     "Dump metadata into a JSON file",
+		Usage:     "Dump metadata into a file",
 		ArgsUsage: "META-URL [FILE]",
 		Description: `
-Dump metadata of the volume in JSON format so users are able to see its content in an easy way.
+Supports two formats: JSON format and binary format.
+1. Dump metadata of the volume in JSON format so users are able to see its content in an easy way.
 Output of this command can be loaded later into an empty database, serving as a method to backup
 metadata or to change metadata engine.
 
@@ -47,6 +49,11 @@ $ juicefs dump redis://localhost meta-dump.json.gz
 # Dump only a subtree of the volume to STDOUT
 $ juicefs dump redis://localhost --subdir /dir/in/jfs
 
+2. Binary format is more compact, faster, and memory-efficient.
+
+Examples:
+$ juicefs dump redis://localhost meta-dump.bin --binary
+
 Details: https://juicefs.com/docs/community/metadata_dump_load`,
 		Flags: []cli.Flag{
 			&cli.StringFlag{
@@ -64,17 +71,21 @@ Details: https://juicefs.com/docs/community/metadata_dump_load`,
 			},
 			&cli.BoolFlag{
 				Name:  "fast",
-				Usage: "speedup dump by load all metadata into memory",
+				Usage: "speedup dump by load all metadata into memory (only works with JSON format and DB/KV engine)",
 			},
 			&cli.BoolFlag{
 				Name:  "skip-trash",
 				Usage: "skip files in trash",
 			},
+			&cli.BoolFlag{
+				Name:  "binary",
+				Usage: "dump metadata into a binary file (different from original JSON format, subdir/fast/skip-trash will be ignored)",
+			},
 		},
 	}
 }
 
-func dumpMeta(m meta.Meta, dst string, threads int, keepSecret, fast, skipTrash bool) (err error) {
+func dumpMeta(m meta.Meta, dst string, threads int, keepSecret, fast, skipTrash, isBinary bool) (err error) {
 	var w io.WriteCloser
 	if dst == "" {
 		w = os.Stdout
@@ -107,6 +118,23 @@ func dumpMeta(m meta.Meta, dst string, threads int, keepSecret, fast, skipTrash
 			w = fp
 		}
 	}
+	if isBinary {
+		progress := utils.NewProgress(false)
+		defer progress.Done()
+
+		bars := make(map[string]*utils.Bar)
+		for _, name := range meta.SegType2Name {
+			bars[name] = progress.AddCountSpinner(name)
+		}
+
+		return m.DumpMetaV2(meta.Background(), w, &meta.DumpOption{
+			KeepSecret: keepSecret,
+			Threads:    threads,
+			Progress: func(name string, cnt int) {
+				bars[name].IncrBy(cnt)
+			},
+		})
+	}
 	return m.DumpMeta(w, 1, threads, keepSecret, fast, skipTrash)
 }
 
@@ -134,7 +162,8 @@ func dump(ctx *cli.Context) error {
 		logger.Warnf("Invalid threads number %d, reset to 1", threads)
 		threads = 1
 	}
-	err := dumpMeta(m, dst, threads, ctx.Bool("keep-secret-key"), ctx.Bool("fast"), ctx.Bool("skip-trash"))
+
+	err := dumpMeta(m, dst, threads, ctx.Bool("keep-secret-key"), ctx.Bool("fast"), ctx.Bool("skip-trash"), ctx.Bool("binary"))
 	if err == nil {
 		if dst == "" {
 			dst = "STDOUT"

cmd/load.go

@@ -20,6 +20,7 @@ import (
 	"compress/gzip"
 	"crypto/x509"
 	"encoding/pem"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -28,6 +29,7 @@ import (
 
 	"github.com/DataDog/zstd"
 	"github.com/juicedata/juicefs/pkg/object"
+	"github.com/olekukonko/tablewriter"
 
 	"github.com/juicedata/juicefs/pkg/meta"
 	"github.com/juicedata/juicefs/pkg/utils"
@@ -49,24 +51,44 @@ func cmdLoad() *cli.Command {
 				Usage: "encrypt algorithm (aes256gcm-rsa, chacha20-rsa)",
 				Value: object.AES256GCM_RSA,
 			},
+			&cli.BoolFlag{
+				Name:  "binary",
+				Usage: "load metadata from a binary file (different from original JSON format)",
+			},
+			&cli.BoolFlag{
+				Name:  "stat",
+				Usage: "show statistics of the metadata binary file",
+			},
+			&cli.IntFlag{
+				Name:  "threads",
+				Value: 10,
+				Usage: "number of threads to load binary metadata, only works with --binary",
+			},
 		},
-		Usage:     "Load metadata from a previously dumped JSON file",
+		Usage:     "Load metadata from a previously dumped file",
 		ArgsUsage: "META-URL [FILE]",
 		Description: `
-Load metadata into an empty metadata engine.
+Load metadata into an empty metadata engine or show statistics of the backup file.
 
 WARNING: Do NOT use new engine and the old one at the same time, otherwise it will probably break
 consistency of the volume.
 
 Examples:
 $ juicefs load redis://localhost/1 meta-dump.json.gz
+$ juicefs load redis://localhost/1 meta-dump.bin --binary --threads 10
+$ juicefs load meta-dump.bin --binary --stat
 
 Details: https://juicefs.com/docs/community/metadata_dump_load`,
 	}
 }
 
 func load(ctx *cli.Context) error {
 	setup(ctx, 1)
+
+	if ctx.Bool("binary") && ctx.Bool("stat") {
+		return statBak(ctx)
+	}
+
 	metaUri := ctx.Args().Get(0)
 	src := ctx.Args().Get(1)
 	removePassword(metaUri)
@@ -132,10 +154,30 @@ func load(ctx *cli.Context) error {
 	}
 	m := meta.NewClient(metaUri, nil)
 	if format, err := m.Load(false); err == nil {
-		return fmt.Errorf("Database %s is used by volume %s", utils.RemovePassword(metaUri), format.Name)
+		return fmt.Errorf("database %s is used by volume %s", utils.RemovePassword(metaUri), format.Name)
 	}
-	if err := m.LoadMeta(r); err != nil {
-		return err
+
+	if ctx.Bool("binary") {
+		progress := utils.NewProgress(false)
+		bars := make(map[string]*utils.Bar)
+		for _, name := range meta.SegType2Name {
+			bars[name] = progress.AddCountSpinner(name)
+		}
+
+		opt := &meta.LoadOption{
+			Threads: ctx.Int("threads"),
+			Progress: func(name string, cnt int) {
+				bars[name].IncrBy(cnt)
+			},
+		}
+		if err := m.LoadMetaV2(meta.WrapContext(ctx.Context), r, opt); err != nil {
+			return err
+		}
+		progress.Done()
+	} else {
+		if err := m.LoadMeta(r); err != nil {
+			return err
+		}
 	}
 	if format, err := m.Load(true); err == nil {
 		if format.SecretKey == "removed" {
@@ -147,3 +189,35 @@ func load(ctx *cli.Context) error {
 	logger.Infof("Load metadata from %s succeed", src)
 	return nil
 }
+
+func statBak(ctx *cli.Context) error {
+	path := ctx.Args().Get(0)
+	if path == "" {
+		return errors.New("missing file path")
+	}
+
+	fp, err := os.Open(path)
+	if err != nil {
+		return fmt.Errorf("failed to open file %s: %w", path, err)
+	}
+	bak := &meta.BakFormat{}
+	footer, err := bak.ReadFooter(fp)
+	if err != nil {
+		return fmt.Errorf("failed to read footer: %w", err)
+	}
+
+	fmt.Printf("Backup Version: %d\n", footer.Msg.Version)
+	data := make([][]string, 0, len(footer.Msg.Infos))
+	for name, info := range footer.Msg.Infos {
+		data = append(data, []string{name, fmt.Sprintf("%d", info.Num), fmt.Sprintf("%+v", info.Offset)})
+	}
+
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Name", "Num", "Offset in File"})
+
+	for _, v := range data {
+		table.Append(v)
+	}
+	table.Render()
+	return nil
+}

go.mod

@@ -46,6 +46,7 @@ require (
 	github.com/minio/minio v0.0.0-20210206053228-97fe57bba92c
 	github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78
 	github.com/ncw/swift/v2 v2.0.1
+	github.com/olekukonko/tablewriter v0.0.1
 	github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/sftp v1.13.5

go.sum

@@ -668,6 +668,7 @@ github.com/nrdcg/namesilo v0.2.1/go.mod h1:lwMvfQTyYq+BbjJd30ylEG4GPSS6PII0Tia4r
 github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88=
 github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/oliverisaac/shellescape v0.0.0-20220131224704-1b6c6b87b668 h1:WUilXdVrxYH+fFkmstviAOj1o9CfoW5O/Sd0LWPIVUA=
 github.com/oliverisaac/shellescape v0.0.0-20220131224704-1b6c6b87b668/go.mod h1:EDgl+cvbmeOQUMTTH94gjXVtFHr8xDe5BiXhWn7Hf1E=