diff --git a/builder/src/optimize_prefetch.rs b/builder/src/optimize_prefetch.rs index b7981485910..b8a7b10583d 100644 --- a/builder/src/optimize_prefetch.rs +++ b/builder/src/optimize_prefetch.rs @@ -8,6 +8,7 @@ use crate::BlobManager; use crate::Bootstrap; use crate::BootstrapManager; use crate::BuildContext; +use crate::BuildOutput; use crate::ChunkSource; use crate::ConversionType; use crate::NodeChunk; @@ -18,7 +19,6 @@ use crate::TreeNode; use anyhow::Context; use anyhow::{Ok, Result}; use nydus_api::ConfigV2; -use nydus_rafs::metadata::layout::v6::RafsV6BlobTable; use nydus_rafs::metadata::layout::RafsBlobTable; use nydus_rafs::metadata::RafsSuper; use nydus_rafs::metadata::RafsVersion; @@ -28,8 +28,7 @@ use nydus_storage::meta::BlobChunkInfoV1Ondisk; use nydus_utils::compress; use sha2::Digest; use std::fs::File; -use std::io::Read; -use std::io::Seek; +use std::io::{Read, Seek, Write}; use std::mem::size_of; use std::sync::Arc; pub struct OptimizePrefetch {} @@ -52,6 +51,7 @@ impl PrefetchBlobState { ctx.blob_features, ); blob_info.set_compressor(ctx.compressor); + blob_info.set_separated_with_prefetch_files_feature(true); let mut blob_ctx = BlobContext::from(ctx, &blob_info, ChunkSource::Build)?; blob_ctx.blob_meta_info_enabled = true; let blob_writer = ArtifactWriter::new(crate::ArtifactStorage::FileDir( @@ -72,13 +72,16 @@ impl OptimizePrefetch { tree: &mut Tree, ctx: &mut BuildContext, bootstrap_mgr: &mut BootstrapManager, - blob_table: &mut RafsV6BlobTable, + blob_table: &mut RafsBlobTable, blobs_dir_path: PathBuf, prefetch_nodes: Vec, - ) -> Result<()> { + ) -> Result { // create a new blob for prefetch layer - let blob_layer_num = blob_table.entries.len(); + let blob_layer_num = match blob_table { + RafsBlobTable::V5(table) => table.get_all().len(), + RafsBlobTable::V6(table) => table.get_all().len(), + }; let mut blob_state = PrefetchBlobState::new(&ctx, blob_layer_num as u32, &blobs_dir_path)?; let mut batch = BatchContextGenerator::new(0)?; for node in &prefetch_nodes { @@ -92,19 +95,19 @@ impl OptimizePrefetch { )?; } - Self::dump_blob(ctx, blob_table, &mut blob_state)?; + let blob_mgr = Self::dump_blob(ctx, blob_table, &mut blob_state)?; debug!("prefetch blob id: {}", ctx.blob_id); Self::build_dump_bootstrap(tree, ctx, bootstrap_mgr, blob_table)?; - Ok(()) + BuildOutput::new(&blob_mgr, &bootstrap_mgr.bootstrap_storage) } fn build_dump_bootstrap( tree: &mut Tree, ctx: &mut BuildContext, bootstrap_mgr: &mut BootstrapManager, - blob_table: &mut RafsV6BlobTable, + blob_table: &mut RafsBlobTable, ) -> Result<()> { let mut bootstrap_ctx = bootstrap_mgr.create_ctx()?; let mut bootstrap = Bootstrap::new(tree.clone())?; @@ -112,46 +115,33 @@ impl OptimizePrefetch { // Build bootstrap bootstrap.build(ctx, &mut bootstrap_ctx)?; - // Verify and update prefetch blob - assert!( - blob_table - .entries - .iter() - .filter(|blob| blob.blob_id() == "prefetch-blob") - .count() - == 1, - "Expected exactly one prefetch-blob" - ); - - // Rewrite prefetch blob id - blob_table - .entries - .iter_mut() - .filter(|blob| blob.blob_id() == "prefetch-blob") - .for_each(|blob| { - let mut info = (**blob).clone(); - info.set_blob_id(ctx.blob_id.clone()); - *blob = Arc::new(info); - }); - - // Dump bootstrap - let blob_table_withprefetch = RafsBlobTable::V6(blob_table.clone()); + let blob_table_withprefetch = match blob_table { + RafsBlobTable::V5(table) => RafsBlobTable::V5(table.clone()), + RafsBlobTable::V6(table) => RafsBlobTable::V6(table.clone()), + }; bootstrap.dump( ctx, &mut bootstrap_mgr.bootstrap_storage, &mut bootstrap_ctx, &blob_table_withprefetch, )?; - Ok(()) } fn dump_blob( ctx: &mut BuildContext, - blob_table: &mut RafsV6BlobTable, + blob_table: &mut RafsBlobTable, blob_state: &mut PrefetchBlobState, - ) -> Result<()> { - blob_table.entries.push(blob_state.blob_info.clone().into()); + ) -> Result { + match blob_table { + RafsBlobTable::V5(table) => { + table.entries.push(blob_state.blob_info.clone().into()); + } + RafsBlobTable::V6(table) => { + table.entries.push(blob_state.blob_info.clone().into()); + } + } + let mut blob_mgr = BlobManager::new(ctx.digester); blob_mgr.add_blob(blob_state.blob_ctx.clone()); blob_mgr.set_current_blob_index(0); @@ -168,7 +158,31 @@ impl OptimizePrefetch { .1 .blob_id .clone(); - Ok(()) + + let entries = match blob_table { + RafsBlobTable::V5(table) => table.get_all(), + RafsBlobTable::V6(table) => table.get_all(), + }; + + // Verify and update prefetch blob + assert!( + entries + .iter() + .filter(|blob| blob.blob_id() == "prefetch-blob") + .count() + == 1, + "Expected exactly one prefetch-blob" + ); + // Rewrite prefetch blob id + match blob_table { + RafsBlobTable::V5(table) => { + rewrite_blob_id(&mut table.entries, "prefetch-blob", ctx.blob_id.clone()) + } + RafsBlobTable::V6(table) => { + rewrite_blob_id(&mut table.entries, "prefetch-blob", ctx.blob_id.clone()) + } + } + Ok(blob_mgr) } fn process_prefetch_node( @@ -176,7 +190,7 @@ impl OptimizePrefetch { node: &TreeNode, prefetch_state: &mut PrefetchBlobState, batch: &mut BatchContextGenerator, - blob_table: &RafsV6BlobTable, + blob_table: &RafsBlobTable, blobs_dir_path: &Path, ) -> Result<()> { let tree_node = tree @@ -184,14 +198,17 @@ impl OptimizePrefetch { .ok_or(anyhow!("failed to get node"))? .node .as_ref(); + let entries = match blob_table { + RafsBlobTable::V5(table) => table.get_all(), + RafsBlobTable::V6(table) => table.get_all(), + }; let blob_id = tree_node .borrow() .chunks .first() - .and_then(|chunk| blob_table.entries.get(chunk.inner.blob_index() as usize)) + .and_then(|chunk| entries.get(chunk.inner.blob_index() as usize).cloned()) .map(|entry| entry.blob_id()) .ok_or(anyhow!("failed to get blob id"))?; - let mut blob_file = Arc::new(File::open(blobs_dir_path.join(blob_id))?); tree_node.borrow_mut().layer_idx = prefetch_state.blob_info.blob_index() as u16; @@ -247,6 +264,17 @@ impl OptimizePrefetch { } } +fn rewrite_blob_id(entries: &mut [Arc], blob_id: &str, new_blob_id: String) { + entries + .iter_mut() + .filter(|blob| blob.blob_id() == blob_id) + .for_each(|blob| { + let mut info = (**blob).clone(); + info.set_blob_id(new_blob_id.clone()); + *blob = Arc::new(info); + }); +} + pub fn update_ctx_from_bootstrap( ctx: &mut BuildContext, config: Arc, diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index 82d597778ea..109dd65c4ff 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -16,6 +16,8 @@ import ( "runtime" "strings" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/optimizer" + "github.com/containerd/containerd/reference/docker" "github.com/distribution/reference" "github.com/dustin/go-humanize" @@ -1160,6 +1162,97 @@ func main() { return copier.Copy(context.Background(), opt) }, }, + { + Name: "optimize", + Usage: "Optimize a source nydus image and push to the target", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "source", + Required: true, + Usage: "Source (Nydus) image reference", + EnvVars: []string{"SOURCE"}, + }, + &cli.StringFlag{ + Name: "target", + Required: true, + Usage: "Target (Nydus) image reference", + EnvVars: []string{"TARGET"}, + }, + &cli.BoolFlag{ + Name: "source-insecure", + Required: false, + Usage: "Skip verifying server certs for HTTPS source registry", + EnvVars: []string{"SOURCE_INSECURE"}, + }, + &cli.BoolFlag{ + Name: "target-insecure", + Required: false, + Usage: "Skip verifying server certs for HTTPS target registry", + EnvVars: []string{"TARGET_INSECURE"}, + }, + + &cli.StringFlag{ + Name: "policy", + Value: "separated-blob-with-prefetch-files", + Usage: "Specify the optimizing way", + EnvVars: []string{"OPTIMIZE_POLICY"}, + }, + &cli.StringFlag{ + Name: "prefetch-files", + Required: false, + Usage: "File path to include prefetch files for optimization", + EnvVars: []string{"PREFETCH_FILES"}, + }, + + &cli.StringFlag{ + Name: "work-dir", + Value: "./tmp", + Usage: "Working directory for image optimization", + EnvVars: []string{"WORK_DIR"}, + }, + + &cli.StringFlag{ + Name: "nydus-image", + Value: "nydus-image", + Usage: "Path to the nydus-image binary, default to search in PATH", + EnvVars: []string{"NYDUS_IMAGE"}, + }, + + &cli.StringFlag{ + Name: "push-chunk-size", + Value: "0MB", + Usage: "Chunk size for pushing a blob layer in chunked", + }, + }, + Action: func(c *cli.Context) error { + setupLogLevel(c) + + pushChunkSize, err := humanize.ParseBytes(c.String("push-chunk-size")) + if err != nil { + return errors.Wrap(err, "invalid --push-chunk-size option") + } + if pushChunkSize > 0 { + logrus.Infof("will push layer with chunk size %s", c.String("push-chunk-size")) + } + opt := optimizer.Opt{ + WorkDir: c.String("work-dir"), + NydusImagePath: c.String("nydus-image"), + + Source: c.String("source"), + Target: c.String("target"), + SourceInsecure: c.Bool("source-insecure"), + TargetInsecure: c.Bool("target-insecure"), + + AllPlatforms: c.Bool("all-platforms"), + Platforms: c.String("platform"), + + PushChunkSize: int64(pushChunkSize), + PrefetchFilesPath: c.String("prefetch-files"), + } + + return optimizer.Optimize(context.Background(), opt) + }, + }, { Name: "commit", Usage: "Create and push a new nydus image from a container's changes that use a nydus image", diff --git a/contrib/nydusify/pkg/optimizer/builder.go b/contrib/nydusify/pkg/optimizer/builder.go new file mode 100644 index 00000000000..66f26ac23d5 --- /dev/null +++ b/contrib/nydusify/pkg/optimizer/builder.go @@ -0,0 +1,87 @@ +package optimizer + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +var logger = logrus.WithField("module", "optimizer") + +func isSignalKilled(err error) bool { + return strings.Contains(err.Error(), "signal: killed") +} + +type BuildOption struct { + BuilderPath string + PrefetchFilesPath string + BootstrapPath string + BlobDir string + OutputBootstrapPath string + OutputJSONPath string + Timeout *time.Duration +} + +type outputJSON struct { + Blobs []string `json:"blobs"` +} + +func Build(option BuildOption) (string, error) { + outputJSONPath := option.OutputJSONPath + args := []string{ + "optimize", + "--log-level", + "warn", + "--prefetch-files", + option.PrefetchFilesPath, + "--bootstrap", + option.BootstrapPath, + "--blob-dir", + option.BlobDir, + "--output-bootstrap", + option.OutputBootstrapPath, + "--output-json", + outputJSONPath, + } + + ctx := context.Background() + var cancel context.CancelFunc + if option.Timeout != nil { + ctx, cancel = context.WithTimeout(ctx, *option.Timeout) + defer cancel() + } + logrus.Debugf("\tCommand: %s %s", option.BuilderPath, strings.Join(args, " ")) + + cmd := exec.CommandContext(ctx, option.BuilderPath, args...) + cmd.Stdout = logger.Writer() + cmd.Stderr = logger.Writer() + + if err := cmd.Run(); err != nil { + if isSignalKilled(err) && option.Timeout != nil { + logrus.WithError(err).Errorf("fail to run %v %+v, possibly due to timeout %v", option.BuilderPath, args, *option.Timeout) + } else { + logrus.WithError(err).Errorf("fail to run %v %+v", option.BuilderPath, args) + } + return "", errors.Wrap(err, "run merge command") + } + + outputBytes, err := os.ReadFile(outputJSONPath) + if err != nil { + return "", errors.Wrapf(err, "read file %s", outputJSONPath) + } + var output outputJSON + err = json.Unmarshal(outputBytes, &output) + if err != nil { + return "", errors.Wrapf(err, "unmarshal output json file %s", outputJSONPath) + } + blobID := output.Blobs[len(output.Blobs)-1] + + logrus.Infof("build success for prefetch blob : %s", blobID) + return blobID, nil +} diff --git a/contrib/nydusify/pkg/optimizer/optimizer.go b/contrib/nydusify/pkg/optimizer/optimizer.go new file mode 100644 index 00000000000..84d35614e5f --- /dev/null +++ b/contrib/nydusify/pkg/optimizer/optimizer.go @@ -0,0 +1,537 @@ +// Copyright 2024 Nydus Developers. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package optimizer + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/goharbor/acceleration-service/pkg/platformutil" + + "github.com/containerd/containerd/content/local" + "github.com/containerd/containerd/reference/docker" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/namespaces" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/committer" + converterpvd "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/converter/provider" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/parser" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/provider" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/remote" + "github.com/dragonflyoss/nydus/contrib/nydusify/pkg/utils" + accerr "github.com/goharbor/acceleration-service/pkg/errdefs" + accremote "github.com/goharbor/acceleration-service/pkg/remote" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +const ( + EntryBootstrap = "image.boot" + EntryPrefetchFiles = "prefetch.files" +) + +type Opt struct { + WorkDir string + NydusImagePath string + + Source string + Target string + + SourceInsecure bool + TargetInsecure bool + + OptimizePolicy string + PrefetchFilesPath string + + AllPlatforms bool + Platforms string + + PushChunkSize int64 +} + +// the information generated during building +type BuildInfo struct { + SourceImage parser.Image + BuildDir string + BlobDir string + PrefetchBlobID string + NewBootstrapPath string +} + +type File struct { + Name string + Reader io.Reader + Size int64 +} + +type bootstrapInfo struct { + bootstrapDesc ocispec.Descriptor + bootstrapDiffID digest.Digest +} + +func hosts(opt Opt) accremote.HostFunc { + maps := map[string]bool{ + opt.Source: opt.SourceInsecure, + opt.Target: opt.TargetInsecure, + } + return func(ref string) (accremote.CredentialFunc, bool, error) { + return accremote.NewDockerConfigCredFunc(), maps[ref], nil + } +} + +func remoter(opt Opt) (*remote.Remote, error) { + targetRef, err := committer.ValidateRef(opt.Target) + if err != nil { + return nil, errors.Wrap(err, "validate target reference") + } + remoter, err := provider.DefaultRemote(targetRef, opt.TargetInsecure) + if err != nil { + return nil, errors.Wrap(err, "create remote") + } + return remoter, nil +} + +func makeDesc(x interface{}, oldDesc ocispec.Descriptor) ([]byte, *ocispec.Descriptor, error) { + data, err := json.MarshalIndent(x, "", " ") + if err != nil { + return nil, nil, errors.Wrap(err, "json marshal") + } + dgst := digest.SHA256.FromBytes(data) + + newDesc := oldDesc + newDesc.Size = int64(len(data)) + newDesc.Digest = dgst + + return data, &newDesc, nil +} + +// packToTar packs files to .tar(.gz) stream then return reader. +// +// ported from https://github.com/containerd/nydus-snapshotter/blob/5f948e4498151b51c742d2ee0b3f7b96f86a26f7/pkg/converter/utils.go#L92 +func packToTar(files []File, compress bool) io.ReadCloser { + dirHdr := &tar.Header{ + Name: "image", + Mode: 0755, + Typeflag: tar.TypeDir, + } + + pr, pw := io.Pipe() + + go func() { + // Prepare targz writer + var tw *tar.Writer + var gw *gzip.Writer + var err error + + if compress { + gw = gzip.NewWriter(pw) + tw = tar.NewWriter(gw) + } else { + tw = tar.NewWriter(pw) + } + + defer func() { + err1 := tw.Close() + var err2 error + if gw != nil { + err2 = gw.Close() + } + + var finalErr error + + // Return the first error encountered to the other end and ignore others. + switch { + case err != nil: + finalErr = err + case err1 != nil: + finalErr = err1 + case err2 != nil: + finalErr = err2 + } + + pw.CloseWithError(finalErr) + }() + + // Write targz stream + if err = tw.WriteHeader(dirHdr); err != nil { + return + } + + for _, file := range files { + hdr := tar.Header{ + Name: filepath.Join("image", file.Name), + Mode: 0444, + Size: file.Size, + } + if err = tw.WriteHeader(&hdr); err != nil { + return + } + if _, err = io.Copy(tw, file.Reader); err != nil { + return + } + } + }() + + return pr +} + +func getOriginalBlobLayers(nydusImage parser.Image) []ocispec.Descriptor { + originalBlobLayers := []ocispec.Descriptor{} + for idx := range nydusImage.Manifest.Layers { + layer := nydusImage.Manifest.Layers[idx] + if layer.MediaType == utils.MediaTypeNydusBlob { + originalBlobLayers = append(originalBlobLayers, layer) + } + } + return originalBlobLayers +} + +func fetchBlobs(ctx context.Context, opt Opt, buildDir string) error { + logrus.Infof("pulling source image") + start := time.Now() + platformMC, err := platformutil.ParsePlatforms(opt.AllPlatforms, opt.Platforms) + if err != nil { + return err + } + pvd, err := converterpvd.New(buildDir, hosts(opt), 200, "v1", platformMC, opt.PushChunkSize) + if err != nil { + return err + } + + sourceNamed, err := docker.ParseDockerRef(opt.Source) + if err != nil { + return errors.Wrap(err, "parse source reference") + } + source := sourceNamed.String() + + if err := pvd.Pull(ctx, source); err != nil { + if accerr.NeedsRetryWithHTTP(err) { + pvd.UsePlainHTTP() + if err := pvd.Pull(ctx, source); err != nil { + return errors.Wrap(err, "try to pull image") + } + } else { + return errors.Wrap(err, "pull source image") + } + } + logrus.Infof("pulled source image, elapsed: %s", time.Since(start)) + return nil +} + +// Optimize coverts and push a new optimized nydus image +func Optimize(ctx context.Context, opt Opt) error { + ctx = namespaces.WithNamespace(ctx, "nydusify") + + sourceRemote, err := provider.DefaultRemote(opt.Source, opt.SourceInsecure) + if err != nil { + return errors.Wrap(err, "Init source image parser") + } + sourceParser, err := parser.New(sourceRemote, runtime.GOARCH) + if err != nil { + return errors.Wrap(err, "failed to create parser") + } + + sourceParsed, err := sourceParser.Parse(ctx) + if err != nil { + return errors.Wrap(err, "parse source image") + } + sourceNydusImage := sourceParsed.NydusImage + + if _, err := os.Stat(opt.WorkDir); err != nil { + if errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(opt.WorkDir, 0755); err != nil { + return errors.Wrap(err, "prepare work directory") + } + // We should only clean up when the work directory not exists + // before, otherwise it may delete user data by mistake. + defer os.RemoveAll(opt.WorkDir) + } else { + return errors.Wrap(err, "stat work directory") + } + } + buildDir, err := os.MkdirTemp(opt.WorkDir, "nydusify-") + if err != nil { + return errors.Wrap(err, "create temp directory") + } + defer os.RemoveAll(buildDir) + + if err := fetchBlobs(ctx, opt, buildDir); err != nil { + return errors.Wrap(err, "prepare nydus blobs") + } + + originalBootstrap := filepath.Join(buildDir, "nydus_bootstrap") + bootstrapDesc := parser.FindNydusBootstrapDesc(&sourceNydusImage.Manifest) + if bootstrapDesc == nil { + return fmt.Errorf("not found Nydus bootstrap layer in manifest") + } + bootstrapReader, err := sourceParser.Remote.Pull(ctx, *bootstrapDesc, true) + if err != nil { + return errors.Wrap(err, "pull Nydus originalBootstrap layer") + } + defer bootstrapReader.Close() + if err := utils.UnpackFile(bootstrapReader, utils.BootstrapFileNameInLayer, originalBootstrap); err != nil { + return errors.Wrap(err, "unpack Nydus originalBootstrap layer") + } + + compressAlgo := bootstrapDesc.Digest.Algorithm().String() + blobDir := filepath.Join(buildDir + "/content/blobs/" + compressAlgo) + outPutJSONPath := filepath.Join(buildDir, "output.json") + newBootstrapPath := filepath.Join(buildDir, "optimized_bootstrap") + builderOpt := BuildOption{ + BuilderPath: opt.NydusImagePath, + PrefetchFilesPath: opt.PrefetchFilesPath, + BootstrapPath: originalBootstrap, + BlobDir: blobDir, + OutputBootstrapPath: newBootstrapPath, + OutputJSONPath: outPutJSONPath, + } + logrus.Infof("begin to build new prefetch blob and bootstrap") + start := time.Now() + prefetchBlobID, err := Build(builderOpt) + if err != nil { + return errors.Wrap(err, "optimize nydus image") + } + logrus.Infof("builded new prefetch blob and bootstrap, elapsed: %s", time.Since(start)) + + buildInfo := BuildInfo{ + SourceImage: *sourceParsed.NydusImage, + BuildDir: buildDir, + BlobDir: blobDir, + PrefetchBlobID: prefetchBlobID, + NewBootstrapPath: newBootstrapPath, + } + + if err := pushNewImage(ctx, opt, buildInfo); err != nil { + return errors.Wrap(err, "push new image") + } + return nil +} + +// push blob +func pushBlob(ctx context.Context, opt Opt, buildInfo BuildInfo) (*ocispec.Descriptor, error) { + blobDir := buildInfo.BlobDir + blobID := buildInfo.PrefetchBlobID + remoter, err := remoter(opt) + if err != nil { + return nil, errors.Wrap(err, "create remote") + } + + blobRa, err := local.OpenReader(filepath.Join(blobDir, blobID)) + if err != nil { + return nil, errors.Wrap(err, "open reader for upper blob") + } + + blobDigest := digest.NewDigestFromEncoded(digest.SHA256, blobID) + blobDesc := ocispec.Descriptor{ + Digest: blobDigest, + Size: blobRa.Size(), + MediaType: utils.MediaTypeNydusBlob, + Annotations: map[string]string{ + utils.LayerAnnotationNydusBlob: "true", + }, + } + + if err := remoter.Push(ctx, blobDesc, true, io.NewSectionReader(blobRa, 0, blobRa.Size())); err != nil { + if utils.RetryWithHTTP(err) { + remoter.MaybeWithHTTP(err) + if err := remoter.Push(ctx, blobDesc, true, io.NewSectionReader(blobRa, 0, blobRa.Size())); err != nil { + return nil, errors.Wrap(err, "push blob") + } + } else { + return nil, errors.Wrap(err, "push blob") + } + } + return &blobDesc, nil +} + +func pushNewBootstrap(ctx context.Context, opt Opt, buildInfo BuildInfo) (*bootstrapInfo, error) { + remoter, err := remoter(opt) + if err != nil { + return nil, errors.Wrap(err, "create remote") + } + bootstrapRa, err := local.OpenReader(buildInfo.NewBootstrapPath) + if err != nil { + return nil, errors.Wrap(err, "open reader for bootstrap") + } + prefetchfilesRa, err := local.OpenReader(opt.PrefetchFilesPath) + if err != nil { + return nil, errors.Wrap(err, "open reader for prefetch files") + } + files := []File{ + { + Name: EntryBootstrap, + Reader: content.NewReader(bootstrapRa), + Size: bootstrapRa.Size(), + }, { + Name: EntryPrefetchFiles, + Reader: content.NewReader(prefetchfilesRa), + Size: prefetchfilesRa.Size(), + }, + } + rc := packToTar(files, false) + defer rc.Close() + + bootstrapTarPath := filepath.Join(buildInfo.BuildDir, "bootstrap.tar") + bootstrapTar, err := os.Create(bootstrapTarPath) + if err != nil { + return nil, errors.Wrap(err, "create bootstrap tar file") + } + defer bootstrapTar.Close() + + tarDigester := digest.SHA256.Digester() + if _, err := io.Copy(io.MultiWriter(bootstrapTar, tarDigester.Hash()), rc); err != nil { + return nil, errors.Wrap(err, "get tar digest") + } + bootstrapDiffID := tarDigester.Digest() + + bootstrapTarRa, err := os.Open(bootstrapTarPath) + if err != nil { + return nil, errors.Wrap(err, "open bootstrap tar file") + } + defer bootstrapTarRa.Close() + + bootstrapTarGzPath := filepath.Join(buildInfo.BuildDir, "bootstrap.tar.gz") + bootstrapTarGz, err := os.Create(bootstrapTarGzPath) + if err != nil { + return nil, errors.Wrap(err, "create bootstrap tar.gz file") + } + defer bootstrapTarGz.Close() + gzDigester := digest.SHA256.Digester() + gzWriter := gzip.NewWriter(io.MultiWriter(bootstrapTarGz, gzDigester.Hash())) + if _, err := io.Copy(gzWriter, bootstrapTarRa); err != nil { + return nil, errors.Wrap(err, "compress bootstrap & prefetchfiles to tar.gz") + } + if err := gzWriter.Close(); err != nil { + return nil, errors.Wrap(err, "close gzip writer") + } + + bootstrapTarGzRa, err := local.OpenReader(bootstrapTarGzPath) + if err != nil { + return nil, errors.Wrap(err, "open reader for upper blob") + } + defer bootstrapTarGzRa.Close() + + oldBootstrapDesc := parser.FindNydusBootstrapDesc(&buildInfo.SourceImage.Manifest) + if oldBootstrapDesc == nil { + return nil, fmt.Errorf("not found originial Nydus bootstrap layer in manifest") + } + + annotations := oldBootstrapDesc.Annotations + annotations[utils.LayerAnnotationNyudsPrefetchBlob] = buildInfo.PrefetchBlobID + + // push bootstrap + bootstrapDesc := ocispec.Descriptor{ + Digest: gzDigester.Digest(), + Size: bootstrapTarGzRa.Size(), + MediaType: ocispec.MediaTypeImageLayerGzip, + Annotations: annotations, + } + + bootstrapRc, err := os.Open(bootstrapTarGzPath) + if err != nil { + return nil, errors.Wrapf(err, "open bootstrap %s", bootstrapTarGzPath) + } + defer bootstrapRc.Close() + if err := remoter.Push(ctx, bootstrapDesc, true, bootstrapRc); err != nil { + return nil, errors.Wrap(err, "push bootstrap layer") + } + return &bootstrapInfo{ + bootstrapDesc: bootstrapDesc, + bootstrapDiffID: bootstrapDiffID, + }, nil +} + +func pushConfig(ctx context.Context, opt Opt, buildInfo BuildInfo, bootstrapDiffID digest.Digest) (*ocispec.Descriptor, error) { + nydusImage := buildInfo.SourceImage + remoter, err := remoter(opt) + if err != nil { + return nil, errors.Wrap(err, "create remote") + } + config := nydusImage.Config + + originalBlobLayers := getOriginalBlobLayers(nydusImage) + config.RootFS.DiffIDs = []digest.Digest{} + for idx := range originalBlobLayers { + config.RootFS.DiffIDs = append(config.RootFS.DiffIDs, originalBlobLayers[idx].Digest) + } + prefetchBlobDigest := digest.NewDigestFromEncoded(digest.SHA256, buildInfo.PrefetchBlobID) + config.RootFS.DiffIDs = append(config.RootFS.DiffIDs, prefetchBlobDigest) + // Note: bootstrap diffid is tar + config.RootFS.DiffIDs = append(config.RootFS.DiffIDs, bootstrapDiffID) + + configBytes, configDesc, err := makeDesc(config, nydusImage.Manifest.Config) + if err != nil { + return nil, errors.Wrap(err, "make config desc") + } + + if err := remoter.Push(ctx, *configDesc, true, bytes.NewReader(configBytes)); err != nil { + if utils.RetryWithHTTP(err) { + remoter.MaybeWithHTTP(err) + if err := remoter.Push(ctx, *configDesc, true, bytes.NewReader(configBytes)); err != nil { + return nil, errors.Wrap(err, "push image config") + } + } else { + return nil, errors.Wrap(err, "push image config") + } + } + + return configDesc, nil + +} + +func pushNewImage(ctx context.Context, opt Opt, buildInfo BuildInfo) error { + logrus.Infof("pushing new image") + start := time.Now() + + remoter, err := remoter(opt) + if err != nil { + return errors.Wrap(err, "create remote") + } + nydusImage := buildInfo.SourceImage + + prefetchBlob, err := pushBlob(ctx, opt, buildInfo) + if err != nil { + return errors.Wrap(err, "create and push hot blob desc") + } + + bootstrapInfo, err := pushNewBootstrap(ctx, opt, buildInfo) + if err != nil { + return errors.Wrap(err, "create and push bootstrap desc") + } + + configDesc, err := pushConfig(ctx, opt, buildInfo, bootstrapInfo.bootstrapDiffID) + if err != nil { + return errors.Wrap(err, "create and push bootstrap desc") + } + + // push image manifest + layers := getOriginalBlobLayers(nydusImage) + layers = append(layers, *prefetchBlob) + layers = append(layers, bootstrapInfo.bootstrapDesc) + nydusImage.Manifest.Config = *configDesc + nydusImage.Manifest.Layers = layers + + manifestBytes, manifestDesc, err := makeDesc(nydusImage.Manifest, nydusImage.Desc) + if err != nil { + return errors.Wrap(err, "make config desc") + } + if err := remoter.Push(ctx, *manifestDesc, false, bytes.NewReader(manifestBytes)); err != nil { + return errors.Wrap(err, "push image manifest") + } + logrus.Infof("pushed new image, elapsed: %s", time.Since(start)) + return nil +} diff --git a/contrib/nydusify/pkg/utils/constant.go b/contrib/nydusify/pkg/utils/constant.go index 87e6075c85a..78c6ea23eec 100644 --- a/contrib/nydusify/pkg/utils/constant.go +++ b/contrib/nydusify/pkg/utils/constant.go @@ -22,5 +22,6 @@ const ( LayerAnnotationUncompressed = "containerd.io/uncompressed" - LayerAnnotationNydusCommitBlobs = "containerd.io/snapshot/nydus-commit-blobs" + LayerAnnotationNydusCommitBlobs = "containerd.io/snapshot/nydus-commit-blobs" + LayerAnnotationNyudsPrefetchBlob = "containerd.io/snapshot/nydus-separated-blob-with-prefetch-files" ) diff --git a/src/bin/nydus-image/main.rs b/src/bin/nydus-image/main.rs index ad47a4f5047..06bd168dd03 100644 --- a/src/bin/nydus-image/main.rs +++ b/src/bin/nydus-image/main.rs @@ -38,7 +38,6 @@ use nydus_builder::{ TarballBuilder, Tree, TreeNode, WhiteoutSpec, }; -use nydus_rafs::metadata::layout::v6::RafsV6BlobTable; use nydus_rafs::metadata::{MergeError, RafsSuper, RafsSuperConfig, RafsVersion}; use nydus_storage::backend::localfs::LocalFs; use nydus_storage::backend::BlobBackend; @@ -54,6 +53,10 @@ use serde::{Deserialize, Serialize}; use crate::unpack::{OCIUnpacker, Unpacker}; use crate::validator::Validator; +use nydus_rafs::metadata::layout::v5::{RafsV5BlobTable, RafsV5ExtBlobTable}; +use nydus_rafs::metadata::layout::v6::RafsV6BlobTable; + +use nydus_rafs::metadata::layout::RafsBlobTable; #[cfg(target_os = "linux")] use nydus_service::ServiceArgs; @@ -560,7 +563,16 @@ fn prepare_cmd_args(bti_string: &'static str) -> App { .help( "Directory for localfs storage backend, hosting data blobs and cache files", ), - ), + ) + .arg( + Arg::new("output-bootstrap") + .long("output-bootstrap") + .short('O') + .help("Output path of optimized bootstrap"), + ) + .arg( + arg_output_json.clone(), + ) ); #[cfg(target_os = "linux")] @@ -911,7 +923,7 @@ fn main() -> Result<()> { } else if let Some(matches) = cmd.subcommand_matches("unpack") { Command::unpack(matches) } else if let Some(matches) = cmd.subcommand_matches("optimize") { - Command::optimize(matches) + Command::optimize(matches, &build_info) } else { #[cfg(target_os = "linux")] if let Some(matches) = cmd.subcommand_matches("export") { @@ -1670,11 +1682,16 @@ impl Command { Ok(()) } - fn optimize(matches: &ArgMatches) -> Result<()> { + fn optimize(matches: &ArgMatches, build_info: &BuildTimeInfo) -> Result<()> { let blobs_dir_path = Self::get_blobs_dir(matches)?; let prefetch_files = Self::get_prefetch_files(matches)?; prefetch_files.iter().for_each(|f| println!("{}", f)); let bootstrap_path = Self::get_bootstrap(matches)?; + let dst_bootstrap = match matches.get_one::("output-bootstrap") { + None => ArtifactStorage::SingleFile(PathBuf::from("optimized_bootstrap")), + Some(s) => ArtifactStorage::SingleFile(PathBuf::from(s)), + }; + let config = Self::get_configuration(matches)?; config.internal.set_blob_accessible(true); let mut build_ctx = BuildContext { @@ -1695,25 +1712,34 @@ impl Command { } } - let bootstrap_path = ArtifactStorage::SingleFile(PathBuf::from("optimized_bootstrap")); - let mut bootstrap_mgr = BootstrapManager::new(Some(bootstrap_path), None); + let mut bootstrap_mgr = BootstrapManager::new(Some(dst_bootstrap), None); let blobs = sb.superblock.get_blob_infos(); - let mut rafsv6table = RafsV6BlobTable::new(); - for blob in &blobs { - rafsv6table.entries.push(blob.clone()); - } - OptimizePrefetch::generate_prefetch( + let mut blob_table = match build_ctx.fs_version { + RafsVersion::V5 => RafsBlobTable::V5(RafsV5BlobTable { + entries: blobs, + extended: RafsV5ExtBlobTable::new(), + }), + + RafsVersion::V6 => RafsBlobTable::V6(RafsV6BlobTable { entries: blobs }), + }; + let output = OptimizePrefetch::generate_prefetch( &mut tree, &mut build_ctx, &mut bootstrap_mgr, - &mut rafsv6table, + &mut blob_table, blobs_dir_path.to_path_buf(), prefetch_nodes, ) .with_context(|| "Failed to generate prefetch bootstrap")?; - Ok(()) + OutputSerializer::dump( + matches, + output, + build_info, + build_ctx.compressor, + build_ctx.fs_version, + ) } fn inspect(matches: &ArgMatches) -> Result<()> { diff --git a/storage/src/device.rs b/storage/src/device.rs index c8b44347377..3504dc8330a 100644 --- a/storage/src/device.rs +++ b/storage/src/device.rs @@ -79,6 +79,8 @@ bitflags! { const _V5_NO_EXT_BLOB_TABLE = 0x8000_0000; /// Blob is generated with chunkdict. const IS_CHUNKDICT_GENERATED = 0x0000_0200; + /// Blob is generated with separated prefetch files. + const IS_SEPARATED_WITH_PREFETCH_FILES = 0x0000_0400; } } @@ -508,6 +510,12 @@ impl BlobInfo { } } + pub fn set_separated_with_prefetch_files_feature(&mut self, is_prefetchblob: bool) { + if is_prefetchblob { + self.blob_features |= BlobFeatures::IS_SEPARATED_WITH_PREFETCH_FILES; + } + } + /// Get SHA256 digest of the ToC content, including the toc tar header. /// /// It's all zero for inlined bootstrap.