builder: introduce optimize subcommand of nydusify

We can statically analyze the image entrypoint dependency, or use runtime dynamic
analysis technologies such as ebpf, fanotify, metric, etc. to obtain the container
file access pattern, and then build this part of data into an independent image layer:

* preferentially fetch blob during the image startup phase to reduce network and disk IO.
* avoid frequent image builds, allows for better local cache utilization.

Implement optimize subcommand of nydusify to generate a new image, which references a new
blob included prefetch file chunks.
```
nydusify optimize --policy separated-prefetch-blob \
	--source $existed-nydus-image \
	--target $new-nydus-image \
	--prefetch-files /path/to/prefetch-files
```

More detailed process is as follows:
1. nydusify first downloads the source image and bootstrap, utilize nydus-image to output a
new bootstrap along with an independent prefetchblob;
2. nydusify generate&push new meta layer including new bootstrap and the prefetch-files ,
also generates&push new manifest/config/prefetchblob, completing the incremental image build.

Signed-off-by: Xing Ma <[email protected]>

contrib/nydusify/cmd/nydusify.go

@@ -16,6 +16,8 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/dragonflyoss/nydus/contrib/nydusify/pkg/optimizer"
+
 	"github.com/containerd/containerd/reference/docker"
 	"github.com/distribution/reference"
 	"github.com/dustin/go-humanize"
@@ -1160,6 +1162,97 @@ func main() {
 				return copier.Copy(context.Background(), opt)
 			},
 		},
+		{
+			Name:  "optimize",
+			Usage: "Convert and push a new optimized nydus image",
+			Flags: []cli.Flag{
+				&cli.StringFlag{
+					Name:     "source",
+					Required: true,
+					Usage:    "Source (Nydus) image reference",
+					EnvVars:  []string{"SOURCE"},
+				},
+				&cli.StringFlag{
+					Name:     "target",
+					Required: true,
+					Usage:    "Target (Nydus) image reference",
+					EnvVars:  []string{"TARGET"},
+				},
+				&cli.BoolFlag{
+					Name:     "source-insecure",
+					Required: false,
+					Usage:    "Skip verifying server certs for HTTPS source registry",
+					EnvVars:  []string{"SOURCE_INSECURE"},
+				},
+				&cli.BoolFlag{
+					Name:     "target-insecure",
+					Required: false,
+					Usage:    "Skip verifying server certs for HTTPS target registry",
+					EnvVars:  []string{"TARGET_INSECURE"},
+				},
+
+				&cli.StringFlag{
+					Name:    "policy",
+					Value:   "separated-prefetch-blob",
+					Usage:   "Specify the optimizing way",
+					EnvVars: []string{"OPTIMIZE_POLICY"},
+				},
+				&cli.StringFlag{
+					Name:     "prefetch-files",
+					Required: false,
+					Usage:    "Specify the prefetch files to build preftech blobs",
+					EnvVars:  []string{"PREFETCH_FILES"},
+				},
+
+				&cli.StringFlag{
+					Name:    "work-dir",
+					Value:   "./tmp",
+					Usage:   "Working directory for image copy",
+					EnvVars: []string{"WORK_DIR"},
+				},
+
+				&cli.StringFlag{
+					Name:    "nydus-image",
+					Value:   "nydus-image",
+					Usage:   "Path to the nydus-image binary, default to search in PATH",
+					EnvVars: []string{"NYDUS_IMAGE"},
+				},
+
+				&cli.StringFlag{
+					Name:  "push-chunk-size",
+					Value: "0MB",
+					Usage: "Chunk size for pushing a blob layer in chunked",
+				},
+			},
+			Action: func(c *cli.Context) error {
+				setupLogLevel(c)
+
+				pushChunkSize, err := humanize.ParseBytes(c.String("push-chunk-size"))
+				if err != nil {
+					return errors.Wrap(err, "invalid --push-chunk-size option")
+				}
+				if pushChunkSize > 0 {
+					logrus.Infof("will copy layer with chunk size %s", c.String("push-chunk-size"))
+				}
+				opt := optimizer.Opt{
+					WorkDir:        c.String("work-dir"),
+					NydusImagePath: c.String("nydus-image"),
+
+					Source:         c.String("source"),
+					Target:         c.String("target"),
+					SourceInsecure: c.Bool("source-insecure"),
+					TargetInsecure: c.Bool("target-insecure"),
+
+					AllPlatforms: c.Bool("all-platforms"),
+					Platforms:    c.String("platform"),
+
+					PushChunkSize:     int64(pushChunkSize),
+					PrefetchFilesPath: c.String("prefetch-files"),
+				}
+
+				return optimizer.Optimize(context.Background(), opt)
+			},
+		},
 		{
 			Name:  "commit",
 			Usage: "Create and push a new nydus image from a container's changes that use a nydus image",

contrib/nydusify/pkg/optimizer/builder.go

@@ -0,0 +1,80 @@
+package optimizer
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+var logger = logrus.WithField("module", "optimizer")
+
+func isSignalKilled(err error) bool {
+	return strings.Contains(err.Error(), "signal: killed")
+}
+
+type BuildOption struct {
+	BuilderPath       string
+	PrefetchFilesPath string
+	BootstrapPath     string
+	BlobDir           string
+	NewBootstrapPath  string
+	OutputPath        string
+	Timeout           *time.Duration
+}
+
+type outputJSON struct {
+	NewBootstrapPath string
+	PrefetchBlob     string
+}
+
+func Build(option BuildOption) (string, error) {
+	args := []string{
+		"optimize",
+		"--log-level",
+		"warn",
+		"--prefetch-files",
+		option.PrefetchFilesPath,
+		"--bootstrap",
+		option.BootstrapPath,
+		"--blob-dir",
+		option.BlobDir,
+		"--new-bootstrap",
+		option.NewBootstrapPath,
+		"--output-path",
+		option.OutputPath,
+	}
+
+	ctx := context.Background()
+	var cancel context.CancelFunc
+	if option.Timeout != nil {
+		ctx, cancel = context.WithTimeout(ctx, *option.Timeout)
+		defer cancel()
+	}
+	logrus.Debugf("\tCommand: %s %s", option.BuilderPath, strings.Join(args, " "))
+
+	cmd := exec.CommandContext(ctx, option.BuilderPath, args...)
+	cmd.Stdout = logger.Writer()
+	cmd.Stderr = logger.Writer()
+
+	if err := cmd.Run(); err != nil {
+		if isSignalKilled(err) && option.Timeout != nil {
+			logrus.WithError(err).Errorf("fail to run %v %+v, possibly due to timeout %v", option.BuilderPath, args, *option.Timeout)
+		} else {
+			logrus.WithError(err).Errorf("fail to run %v %+v", option.BuilderPath, args)
+		}
+		return "", errors.Wrap(err, "run merge command")
+	}
+
+	BlobId, err := os.ReadFile(option.OutputPath)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to read blob id file\n")
+	}
+
+	logrus.Infof("build success for prefetch blob : %s", BlobId)
+	return string(BlobId), nil
+}