diff --git a/docs/docs.md b/docs/docs.md index 3a177b2..eea72aa 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -73,6 +73,52 @@ Create a tarball and an OCI descriptor for it | symlinks | Dictionary of symlink -> target entries to place in the tarball | Dictionary: String -> String | optional | {} | + + +## oci_image_layout + +
+oci_image_layout(name, manifest, registry, repository)
+
+ + + Writes an OCI Image Index and related blobs to an OCI Image Format + directory. See https://github.com/opencontainers/image-spec/blob/main/image-layout.md + for the specification of the OCI Image Format directory. Local blobs are + used where available, and if a referenced blob is not present, it is + fetched from the provided OCI repository and placed in the output. + + In order for this rule to work correctly in its current state, the + following flags must be provided to bazel: + --incompatible_strict_action_env=false + --spawn_strategy=local + + The incompatible_strict_action_env flag is required because in order to + access the registry, a credential helper executable (named + docker-credential-) must be available so that ocitool can + execute it. The incompatible_strict_action_env flag makes the system + PATH available to bazel rules. + + The spawn_strategy flag must be set to local because currently, + oci_image_index is only declaring the new JSON files it creates as + outputs; it's not declaring any manifests or layers from the images as + outputs. By default, Bazel only permits rules to access specifically + declared outputs of the rule's direct dependencies. In order for this + rule to access the transitive set of outputs of all dependencies, we + must disable bazel's sandboxing by setting spawn_strategy=local. + + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| manifest | An OCILayout index to be written to the OCI Image Format directory. | Label | optional | None | +| registry | A registry host that contains images referenced by the OCILayout index, if not present consult the toolchain. | String | optional | "" | +| repository | A repository that contains images referenced by the OCILayout index, if not present consult the toolchain. | String | optional | "" | + + ## oci_pull diff --git a/go/cmd/ocitool/BUILD.bazel b/go/cmd/ocitool/BUILD.bazel index 0a7de52..85212ff 100644 --- a/go/cmd/ocitool/BUILD.bazel +++ b/go/cmd/ocitool/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "appendlayer_cmd.go", "createlayer_cmd.go", + "imagelayout_cmd.go", "desc_helpers.go", "digest_cmd.go", "gen_cmd.go", diff --git a/go/cmd/ocitool/imagelayout_cmd.go b/go/cmd/ocitool/imagelayout_cmd.go new file mode 100644 index 0000000..dbea508 --- /dev/null +++ b/go/cmd/ocitool/imagelayout_cmd.go @@ -0,0 +1,68 @@ +package main + +import ( + "fmt" + "path" + + "github.com/DataDog/rules_oci/go/pkg/ociutil" + "github.com/containerd/containerd/images" + "github.com/urfave/cli/v2" +) + +// This command creates an OCI Image Layout directory based on the layout parameter. +// See https://github.com/opencontainers/image-spec/blob/main/image-layout.md +// for the structure of OCI Image Layout directories. +func CreateOciImageLayoutCmd(c *cli.Context) error { + registry := c.String("registry") + repository := c.String("repository") + ref := path.Join(registry, repository) + + // Setup an OCI resolver. We need this because the provided input layout + // may not contain all required blobs locally. The missing blobs must be + // loaded from a registry. + resolver := ociutil.DefaultResolver() + // Get the fetcher from the resolver, and convert to a provider. + remoteFetcher, err := resolver.Fetcher(c.Context, ref) + if err != nil { + return err + } + ociProvider := ociutil.FetchertoProvider(remoteFetcher) + + // Load providers that read local files, and create a multiprovider that + // contains all of them, as well as the ociProvider. + providers, err := LoadLocalProviders(c.StringSlice("layout"), c.String("layout-relative")) + if err != nil { + return err + } + // If modifying the code below, ensure ociProvider comes after providers... + // we want to use the local provider if a descriptor is present in both. + providers = append(providers, ociProvider) + multiProvider := ociutil.MultiProvider(providers...) + + descriptorFile := c.String("desc") + baseDesc, err := ociutil.ReadDescriptorFromFile(descriptorFile) + if err != nil { + return fmt.Errorf("failed to read base descriptor: %w", err) + } + + outDir := c.String("out-dir") + ociIngester, err := ociutil.NewOciImageLayoutIngester(outDir) + if err != nil { + return err + } + + // Copy the children first; leave the parent (index) to last. + imagesHandler := images.ChildrenHandler(multiProvider) + err = ociutil.CopyChildrenFromHandler(c.Context, imagesHandler, multiProvider, &ociIngester, baseDesc) + if err != nil { + return fmt.Errorf("failed to copy child content to OCI Image Layout: %w", err) + } + + // copy the parent last (in case of image index) + err = ociutil.CopyContent(c.Context, multiProvider, &ociIngester, baseDesc) + if err != nil { + return fmt.Errorf("failed to copy parent content to OCI Image Layout: %w", err) + } + + return nil +} diff --git a/go/cmd/ocitool/main.go b/go/cmd/ocitool/main.go index 71831a4..2c32a22 100644 --- a/go/cmd/ocitool/main.go +++ b/go/cmd/ocitool/main.go @@ -202,6 +202,32 @@ var app = &cli.App{ }, }, }, + { + Name: "create-oci-image-layout", + Description: `Creates a directory containing an OCI Image Layout based on the input layout, +as described in https://github.com/opencontainers/image-spec/blob/main/image-layout.md.`, + Action: CreateOciImageLayoutCmd, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "layout-relative", + }, + &cli.StringFlag{ + Name: "desc", + }, + &cli.StringFlag{ + Name: "registry", + Usage: "An OCI image registry that will be used to pull missing images referenced by the index.", + }, + &cli.StringFlag{ + Name: "repository", + Usage: "An OCI image repository that will be used to pull missing images referenced by the index.", + }, + &cli.StringFlag{ + Name: "out-dir", + Usage: "The directory that the OCI Image Layout will be written to.", + }, + }, + }, { Name: "push-blob", Hidden: true, diff --git a/go/pkg/ociutil/BUILD.bazel b/go/pkg/ociutil/BUILD.bazel index b2377bb..ed17af3 100644 --- a/go/pkg/ociutil/BUILD.bazel +++ b/go/pkg/ociutil/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "json.go", "manifest.go", "multiprovider.go", + "ociimagelayout.go", "platforms.go", "provider.go", "push.go", diff --git a/go/pkg/ociutil/multiprovider.go b/go/pkg/ociutil/multiprovider.go index 64a31a6..d380ed5 100644 --- a/go/pkg/ociutil/multiprovider.go +++ b/go/pkg/ociutil/multiprovider.go @@ -8,7 +8,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// MultiProvider will read from the first provider that can read the requrested +// MultiProvider will read from the first provider that can read the requested // descriptor. func MultiProvider(providers ...content.Provider) content.Provider { return &multiProvider{ diff --git a/go/pkg/ociutil/ociimagelayout.go b/go/pkg/ociutil/ociimagelayout.go new file mode 100644 index 0000000..6647f09 --- /dev/null +++ b/go/pkg/ociutil/ociimagelayout.go @@ -0,0 +1,175 @@ +package ociutil + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "time" + + "github.com/containerd/containerd/content" + "github.com/opencontainers/go-digest" +) + +const BlobsFolderName = "blobs" +const OciImageIndexMediaType = "application/vnd.oci.image.index.v1+json" +const OciLayoutFileName = "oci-layout" +const OciLayoutFileContent = `{ + "imageLayoutVersion": "1.0.0" +}` +const OciIndexFileName = "index.json" +const ContentFileMode = 0755 + +// OciImageLayoutIngester implements functionality to write data to an OCI +// Image Layout directory (https://github.com/opencontainers/image-spec/blob/main/image-layout.md) +type OciImageLayoutIngester struct { + // The path of the directory containing the OCI Image Layout. + Path string +} + +func NewOciImageLayoutIngester(path string) (OciImageLayoutIngester, error) { + if err := os.MkdirAll(path, ContentFileMode); err != nil { + return OciImageLayoutIngester{}, fmt.Errorf("error creating directory for OciImageLayoutIngester: %v, Err: %w", path, err) + } + return OciImageLayoutIngester{Path: path}, nil +} + +// writer returns a Writer object that will write one entity to the OCI Image Layout. +// Examples are OCI Image Index, an OCI Image Manifest, an OCI Image Config, +// and OCI image TAR/GZIP files. +func (ing *OciImageLayoutIngester) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) { + // Initialize the writer options (for those unfamiliar with this pattern, it's known as the + // "functional options pattern"). + var wOpts content.WriterOpts + for _, o := range opts { + if err := o(&wOpts); err != nil { + return nil, fmt.Errorf("unable to apply WriterOpt to WriterOpts. Err: %w", err) + } + } + status := content.Status{ + Ref: wOpts.Ref, + Offset: 0, + Total: wOpts.Desc.Size, + Expected: wOpts.Desc.Digest, + } + return &OciImageLayoutWriter{Path: ing.Path, Opts: wOpts, Stat: status}, nil +} + +type OciImageLayoutWriter struct { + Path string + Opts content.WriterOpts + Dig digest.Digest + Stat content.Status +} + +// Writes bytes to a file, using the provided write flags. +func writeFile(filePath string, writeFlags int, b []byte) error { + f, err := os.OpenFile(filePath, writeFlags, ContentFileMode) + if err != nil { + return fmt.Errorf("error opening file for write: %v, Err: %w", filePath, err) + } + defer f.Close() + + if _, err = f.Write(b); err != nil { + return fmt.Errorf("error writing file: %v, Err: %w", filePath, err) + } + return nil +} + +func (w *OciImageLayoutWriter) Write(b []byte) (n int, err error) { + firstWrite := w.Stat.StartedAt.IsZero() + if firstWrite { + w.Stat.StartedAt = time.Now() + } + + // A function to get the OS flags used to create a writeable file. + getWriteFlags := func(filePath string) (int, error) { + _, err := os.Stat(filePath) + switch { + case firstWrite && err == nil: + // The file exists and it's first write; Truncate it. + return os.O_WRONLY | os.O_TRUNC, nil + case err == nil: + // The file exists and it's not first write; append to it. + return os.O_WRONLY | os.O_APPEND, nil + case errors.Is(err, os.ErrNotExist): + // The file doesn't exist. Create it. + return os.O_WRONLY | os.O_CREATE, nil + default: + // Something went wrong! + return 0, err + } + } + + var filePath string + if w.Opts.Desc.MediaType == OciImageIndexMediaType { + // This is an OCI Image Index. It gets written to the top level index.json file. + // Write the oci-layout file (a simple canned file required by the standard). + layoutFile := path.Join(w.Path, OciLayoutFileName) + // It's possible (but unlikely) for Write to be called repeatedly for an index file. + // In that case, we'll repeatedly rewrite the oci-layout file, which doesn't hurt, + // because the content is always identical. + if err := os.WriteFile(layoutFile, []byte(OciLayoutFileContent), ContentFileMode); err != nil { + return 0, fmt.Errorf("error writing oci-layout file: %v, Err: %w", layoutFile, err) + } + + // Now write the index.json file. + filePath = path.Join(w.Path, OciIndexFileName) + writeFlags, err := getWriteFlags(filePath) + if err != nil { + return 0, fmt.Errorf("error stat'ing file: %v, Err: %w", filePath, err) + } + err = writeFile(filePath, writeFlags, b) + if err != nil { + return 0, err + } + } else { + // This is a blob. Write it to the blobs folder. + algo := w.Opts.Desc.Digest.Algorithm() + blobDir := path.Join(w.Path, BlobsFolderName, algo.String()) + if err := os.MkdirAll(blobDir, ContentFileMode); err != nil { + return 0, fmt.Errorf("error creating blobDir: %v, Err: %w", blobDir, err) + } + filePath = path.Join(blobDir, w.Opts.Desc.Digest.Encoded()) + + writeFlags, err := getWriteFlags(filePath) + if err != nil { + return 0, fmt.Errorf("error stat'ing file: %v, Err: %w", filePath, err) + } + + // Now write the blob file. + err = writeFile(filePath, writeFlags, b) + if err != nil { + return 0, err + } + } + fInfo, err := os.Stat(filePath) + if err != nil { + return 0, fmt.Errorf("error retrieving FileInfo for file: %v, Err: %w", filePath, err) + } + w.Stat.UpdatedAt = fInfo.ModTime() + return len(b), nil +} + +func (w *OciImageLayoutWriter) Close() error { + return nil +} + +// Returns an empty digest until after Commit is called. +func (w *OciImageLayoutWriter) Digest() digest.Digest { + return w.Dig +} + +func (w *OciImageLayoutWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error { + w.Dig = w.Opts.Desc.Digest + return nil +} + +func (w *OciImageLayoutWriter) Status() (content.Status, error) { + return w.Stat, nil +} + +func (w *OciImageLayoutWriter) Truncate(size int64) error { + return errors.New("truncation is unsupported") +} diff --git a/oci/BUILD.bazel b/oci/BUILD.bazel index 6443160..eff474d 100755 --- a/oci/BUILD.bazel +++ b/oci/BUILD.bazel @@ -38,6 +38,7 @@ bzl_library( visibility = ["//visibility:public"], deps = [ ":image", + ":oci_image_layout", ":pull", ":push", ], @@ -64,6 +65,16 @@ bzl_library( deps = ["@com_github_datadog_rules_oci//oci:providers"], ) +bzl_library( + name = "oci_image_layout", + srcs = ["oci_image_layout.bzl"], + visibility = ["//visibility:public"], + deps = [ + "@com_github_datadog_rules_oci//oci:debug_flag", + "@com_github_datadog_rules_oci//oci:providers", + ], +) + bzl_library( name = "push", srcs = ["push.bzl"], diff --git a/oci/defs.bzl b/oci/defs.bzl index 306f5eb..a57d314 100755 --- a/oci/defs.bzl +++ b/oci/defs.bzl @@ -1,6 +1,7 @@ load(":pull.bzl", _oci_pull = "oci_pull") load(":push.bzl", _oci_push = "oci_push") load(":image.bzl", _oci_image = "oci_image", _oci_image_index = "oci_image_index", _oci_image_layer = "oci_image_layer") +load(":oci_image_layout.bzl", _oci_image_layout = "oci_image_layout") oci_pull = _oci_pull oci_push = _oci_push @@ -8,3 +9,4 @@ oci_push = _oci_push oci_image = _oci_image oci_image_index = _oci_image_index oci_image_layer = _oci_image_layer +oci_image_layout = _oci_image_layout diff --git a/oci/oci_image_layout.bzl b/oci/oci_image_layout.bzl new file mode 100644 index 0000000..ddaf3a5 --- /dev/null +++ b/oci/oci_image_layout.bzl @@ -0,0 +1,99 @@ +load("@com_github_datadog_rules_oci//oci:providers.bzl", "OCIDescriptor", "OCILayout") +load("@com_github_datadog_rules_oci//oci:debug_flag.bzl", "DebugInfo") + +def _oci_image_layout_impl(ctx): + toolchain = ctx.toolchains["@com_github_datadog_rules_oci//oci:toolchain"] + + layout = ctx.attr.manifest[OCILayout] + descriptor = ctx.attr.manifest[OCIDescriptor] + + out_dir = ctx.actions.declare_directory(ctx.label.name) + + ctx.actions.run( + executable = toolchain.sdk.ocitool, + arguments = [ + "--layout={layout}".format(layout = layout.blob_index.path), + "--debug={debug}".format(debug = str(ctx.attr._debug[DebugInfo].debug)), + "create-oci-image-layout", + # We need to use the directory one level above bazel-out for the + # layout-relative directory. This is because the paths in + # oci_image_index's index.layout.json are of the form: + # "bazel-out/os_arch-fastbuild/bin/...". Unfortunately, bazel + # provides no direct way to access this directory, so here we traverse + # up 3 levels from the bin directory. + "--layout-relative={root}".format(root = ctx.bin_dir.path+"/../../../"), + "--desc={desc}".format(desc = descriptor.descriptor_file.path), + "--registry={registry}".format(registry = ctx.attr.registry), + "--repository={repository}".format(repository = ctx.attr.repository), + "--out-dir={out_dir}".format(out_dir = out_dir.path), + ], + inputs = ctx.files.manifest, + outputs = [ + out_dir, + ], + use_default_shell_env = True, + ) + + return DefaultInfo(files = depset([out_dir])) + +oci_image_layout = rule( + doc = """ + Writes an OCI Image Index and related blobs to an OCI Image Format + directory. See https://github.com/opencontainers/image-spec/blob/main/image-layout.md + for the specification of the OCI Image Format directory. Local blobs are + used where available, and if a referenced blob is not present, it is + fetched from the provided OCI repository and placed in the output. + + In order for this rule to work correctly in its current state, the + following flags must be provided to bazel: + --incompatible_strict_action_env=false + --spawn_strategy=local + + The incompatible_strict_action_env flag is required because in order to + access the registry, a credential helper executable (named + docker-credential-) must be available so that ocitool can + execute it. The incompatible_strict_action_env flag makes the system + PATH available to bazel rules. + + The spawn_strategy flag must be set to local because currently, + oci_image_index is only declaring the new JSON files it creates as + outputs; it's not declaring any manifests or layers from the images as + outputs. By default, Bazel only permits rules to access specifically + declared outputs of the rule's direct dependencies. In order for this + rule to access the transitive set of outputs of all dependencies, we + must disable bazel's sandboxing by setting spawn_strategy=local. + """, + # TODO(kim.mason): Fix oci_image/oci_image_index so they explicitly declare + # outputs that include everything needed to build the image. + # TODO(kim.mason): Make it so that Docker credential helpers are available + # to oci_image_layout without making the system PATH available. + implementation = _oci_image_layout_impl, + attrs = { + "manifest": attr.label( + doc = """ + An OCILayout index to be written to the OCI Image Format directory. + """, + providers = [OCILayout], + ), + "registry": attr.string( + doc = """ + A registry host that contains images referenced by the OCILayout index, + if not present consult the toolchain. + """, + ), + "repository": attr.string( + doc = """ + A repository that contains images referenced by the OCILayout index, + if not present consult the toolchain. + """, + ), + "_debug": attr.label( + default = "//oci:debug", + providers = [DebugInfo], + ), + }, + provides = [ + DefaultInfo, + ], + toolchains = ["@com_github_datadog_rules_oci//oci:toolchain"], +) diff --git a/oci/providers.bzl b/oci/providers.bzl index 8057d9a..b6d08d5 100755 --- a/oci/providers.bzl +++ b/oci/providers.bzl @@ -41,7 +41,7 @@ OCIImageManifest = provider( OCIImageIndexManifest = provider( doc = "", fields = { - "manifests": "List of desciptors", + "manifests": "List of descriptors", "annotations": "String map of arbitrary metadata", }, )