[OBSOLETE] Add an OCI Image Layout command to the ociutil tool, and a corresponding Bazel rule.

docs/docs.md

@@ -73,6 +73,52 @@ Create a tarball and an OCI descriptor for it
 | <a id="oci_image_layer-symlinks"></a>symlinks |  Dictionary of symlink -&gt; target entries to place in the tarball   | <a href="https://bazel.build/docs/skylark/lib/dict.html">Dictionary: String -> String</a> | optional | {} |
 
 
+<a id="#oci_image_layout"></a>
+
+## oci_image_layout
+
+<pre>
+oci_image_layout(<a href="#oci_image_layout-name">name</a>, <a href="#oci_image_layout-manifest">manifest</a>, <a href="#oci_image_layout-registry">registry</a>, <a href="#oci_image_layout-repository">repository</a>)
+</pre>
+
+
+        Writes an OCI Image Index and related blobs to an OCI Image Format
+        directory. See https://github.com/opencontainers/image-spec/blob/main/image-layout.md
+        for the specification of the OCI Image Format directory. Local blobs are
+        used where available, and if a referenced blob is not present, it is
+        fetched from the provided OCI repository and placed in the output.
+
+        In order for this rule to work correctly in its current state, the
+        following flags must be provided to bazel:
+        --incompatible_strict_action_env=false
+        --spawn_strategy=local
+
+        The incompatible_strict_action_env flag is required because in order to
+        access the registry, a credential helper executable (named
+        docker-credential-<SOMETHING>) must be available so that ocitool can
+        execute it. The incompatible_strict_action_env flag makes the system
+        PATH available to bazel rules.
+
+        The spawn_strategy flag must be set to local because currently,
+        oci_image_index is only declaring the new JSON files it creates as
+        outputs; it's not declaring any manifests or layers from the images as
+        outputs. By default, Bazel only permits rules to access specifically
+        declared outputs of the rule's direct dependencies. In order for this
+        rule to access the transitive set of outputs of all dependencies, we
+        must disable bazel's sandboxing by setting spawn_strategy=local.
+
+
+**ATTRIBUTES**
+
+
+| Name  | Description | Type | Mandatory | Default |
+| :------------- | :------------- | :------------- | :------------- | :------------- |
+| <a id="oci_image_layout-name"></a>name |  A unique name for this target.   | <a href="https://bazel.build/docs/build-ref.html#name">Name</a> | required |  |
+| <a id="oci_image_layout-manifest"></a>manifest |  An OCILayout index to be written to the OCI Image Format directory.   | <a href="https://bazel.build/docs/build-ref.html#labels">Label</a> | optional | None |
+| <a id="oci_image_layout-registry"></a>registry |  A registry host that contains images referenced by the OCILayout index,                 if not present consult the toolchain.   | String | optional | "" |
+| <a id="oci_image_layout-repository"></a>repository |  A repository that contains images referenced by the OCILayout index,                 if not present consult the toolchain.   | String | optional | "" |
+
+
 <a id="#oci_pull"></a>
 
 ## oci_pull

go/cmd/ocitool/BUILD.bazel

@@ -5,6 +5,7 @@ go_library(
     srcs = [
         "appendlayer_cmd.go",
         "createlayer_cmd.go",
+        "imagelayout_cmd.go",
         "desc_helpers.go",
         "digest_cmd.go",
         "gen_cmd.go",

go/cmd/ocitool/imagelayout_cmd.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"fmt"
+	"path"
+
+	"github.com/DataDog/rules_oci/go/pkg/ociutil"
+	"github.com/containerd/containerd/images"
+	"github.com/urfave/cli/v2"
+)
+
+// This command creates an OCI Image Layout directory based on the layout parameter.
+// See https://github.com/opencontainers/image-spec/blob/main/image-layout.md
+// for the structure of OCI Image Layout directories.
+func CreateOciImageLayoutCmd(c *cli.Context) error {
+	registry := c.String("registry")
+	repository := c.String("repository")
+	ref := path.Join(registry, repository)
+
+	// Setup an OCI resolver. We need this because the provided input layout
+	// may not contain all required blobs locally. The missing blobs must be
+	// loaded from a registry.
+	resolver := ociutil.DefaultResolver()
+	// Get the fetcher from the resolver, and convert to a provider.
+	remoteFetcher, err := resolver.Fetcher(c.Context, ref)
+	if err != nil {
+		return err
+	}
+	ociProvider := ociutil.FetchertoProvider(remoteFetcher)
+
+	// Load providers that read local files, and create a multiprovider that
+	// contains all of them, as well as the ociProvider.
+	providers, err := LoadLocalProviders(c.StringSlice("layout"), c.String("layout-relative"))
+	if err != nil {
+		return err
+	}
+	// If modifying the code below, ensure ociProvider comes after providers...
+	// we want to use the local provider if a descriptor is present in both.
+	providers = append(providers, ociProvider)
+	multiProvider := ociutil.MultiProvider(providers...)
+
+	descriptorFile := c.String("desc")
+	baseDesc, err := ociutil.ReadDescriptorFromFile(descriptorFile)
+	if err != nil {
+		return fmt.Errorf("failed to read base descriptor: %w", err)
+	}
+
+	outDir := c.String("out-dir")
+	ociIngester, err := ociutil.NewOciImageLayoutIngester(outDir)
+	if err != nil {
+		return err
+	}
+
+	// Copy the children first; leave the parent (index) to last.
+	imagesHandler := images.ChildrenHandler(multiProvider)
+	err = ociutil.CopyChildrenFromHandler(c.Context, imagesHandler, multiProvider, &ociIngester, baseDesc)
+	if err != nil {
+		return fmt.Errorf("failed to copy child content to OCI Image Layout: %w", err)
+	}
+
+	// copy the parent last (in case of image index)
+	err = ociutil.CopyContent(c.Context, multiProvider, &ociIngester, baseDesc)
+	if err != nil {
+		return fmt.Errorf("failed to copy parent content to OCI Image Layout: %w", err)
+	}
+
+	return nil
+}

go/cmd/ocitool/main.go

@@ -202,6 +202,32 @@ var app = &cli.App{
 				},
 			},
 		},
+		{
+			Name: "create-oci-image-layout",
+			Description: `Creates a directory containing an OCI Image Layout based on the input layout,
+as described in https://github.com/opencontainers/image-spec/blob/main/image-layout.md.`,
+			Action: CreateOciImageLayoutCmd,
+			Flags: []cli.Flag{
+				&cli.StringFlag{
+					Name: "layout-relative",
+				},
+				&cli.StringFlag{
+					Name: "desc",
+				},
+				&cli.StringFlag{
+					Name:  "registry",
+					Usage: "An OCI image registry that will be used to pull missing images referenced by the index.",
+				},
+				&cli.StringFlag{
+					Name:  "repository",
+					Usage: "An OCI image repository that will be used to pull missing images referenced by the index.",
+				},
+				&cli.StringFlag{
+					Name:  "out-dir",
+					Usage: "The directory that the OCI Image Layout will be written to.",
+				},
+			},
+		},
 		{
 			Name:   "push-blob",
 			Hidden: true,

go/pkg/ociutil/BUILD.bazel

@@ -13,6 +13,7 @@ go_library(
         "json.go",
         "manifest.go",
         "multiprovider.go",
+        "ociimagelayout.go",
         "platforms.go",
         "provider.go",
         "push.go",

go/pkg/ociutil/multiprovider.go

@@ -8,7 +8,7 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-// MultiProvider will read from the first provider that can read the requrested
+// MultiProvider will read from the first provider that can read the requested
 // descriptor.
 func MultiProvider(providers ...content.Provider) content.Provider {
 	return &multiProvider{

go/pkg/ociutil/ociimagelayout.go

@@ -0,0 +1,175 @@
+package ociutil
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"time"
+
+	"github.com/containerd/containerd/content"
+	"github.com/opencontainers/go-digest"
+)
+
+const BlobsFolderName = "blobs"
+const OciImageIndexMediaType = "application/vnd.oci.image.index.v1+json"
+const OciLayoutFileName = "oci-layout"
+const OciLayoutFileContent = `{
+    "imageLayoutVersion": "1.0.0"
+}`
+const OciIndexFileName = "index.json"
+const ContentFileMode = 0755
+
+// OciImageLayoutIngester implements functionality to write data to an OCI
+// Image Layout directory (https://github.com/opencontainers/image-spec/blob/main/image-layout.md)
+type OciImageLayoutIngester struct {
+	// The path of the directory containing the OCI Image Layout.
+	Path string
+}
+
+func NewOciImageLayoutIngester(path string) (OciImageLayoutIngester, error) {
+	if err := os.MkdirAll(path, ContentFileMode); err != nil {
+		return OciImageLayoutIngester{}, fmt.Errorf("error creating directory for OciImageLayoutIngester: %v, Err: %w", path, err)
+	}
+	return OciImageLayoutIngester{Path: path}, nil
+}
+
+// writer returns a Writer object that will write one entity to the OCI Image Layout.
+// Examples are OCI Image Index, an OCI Image Manifest, an OCI Image Config,
+// and OCI image TAR/GZIP files.
+func (ing *OciImageLayoutIngester) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
+	// Initialize the writer options (for those unfamiliar with this pattern, it's known as the
+	// "functional options pattern").
+	var wOpts content.WriterOpts
+	for _, o := range opts {
+		if err := o(&wOpts); err != nil {
+			return nil, fmt.Errorf("unable to apply WriterOpt to WriterOpts. Err: %w", err)
+		}
+	}
+	status := content.Status{
+		Ref:      wOpts.Ref,
+		Offset:   0,
+		Total:    wOpts.Desc.Size,
+		Expected: wOpts.Desc.Digest,
+	}
+	return &OciImageLayoutWriter{Path: ing.Path, Opts: wOpts, Stat: status}, nil
+}
+
+type OciImageLayoutWriter struct {
+	Path string
+	Opts content.WriterOpts
+	Dig  digest.Digest
+	Stat content.Status
+}
+
+// Writes bytes to a file, using the provided write flags.
+func writeFile(filePath string, writeFlags int, b []byte) error {
+	f, err := os.OpenFile(filePath, writeFlags, ContentFileMode)
+	if err != nil {
+		return fmt.Errorf("error opening file for write: %v, Err: %w", filePath, err)
+	}
+	defer f.Close()
+
+	if _, err = f.Write(b); err != nil {
+		return fmt.Errorf("error writing file: %v, Err: %w", filePath, err)
+	}
+	return nil
+}
+
+func (w *OciImageLayoutWriter) Write(b []byte) (n int, err error) {
+	firstWrite := w.Stat.StartedAt.IsZero()
+	if firstWrite {
+		w.Stat.StartedAt = time.Now()
+	}
+
+	// A function to get the OS flags used to create a writeable file.
+	getWriteFlags := func(filePath string) (int, error) {
+		_, err := os.Stat(filePath)
+		switch {
+		case firstWrite && err == nil:
+			// The file exists and it's first write; Truncate it.
+			return os.O_WRONLY | os.O_TRUNC, nil
+		case err == nil:
+			// The file exists and it's not first write; append to it.
+			return os.O_WRONLY | os.O_APPEND, nil
+		case errors.Is(err, os.ErrNotExist):
+			// The file doesn't exist. Create it.
+			return os.O_WRONLY | os.O_CREATE, nil
+		default:
+			// Something went wrong!
+			return 0, err
+		}
+	}
+
+	var filePath string
+	if w.Opts.Desc.MediaType == OciImageIndexMediaType {
+		// This is an OCI Image Index. It gets written to the top level index.json file.
+		// Write the oci-layout file (a simple canned file required by the standard).
+		layoutFile := path.Join(w.Path, OciLayoutFileName)
+		// It's possible (but unlikely) for Write to be called repeatedly for an index file.
+		// In that case, we'll repeatedly rewrite the oci-layout file, which doesn't hurt,
+		// because the content is always identical.
+		if err := os.WriteFile(layoutFile, []byte(OciLayoutFileContent), ContentFileMode); err != nil {
+			return 0, fmt.Errorf("error writing oci-layout file: %v, Err: %w", layoutFile, err)
+		}
+
+		// Now write the index.json file.
+		filePath = path.Join(w.Path, OciIndexFileName)
+		writeFlags, err := getWriteFlags(filePath)
+		if err != nil {
+			return 0, fmt.Errorf("error stat'ing file: %v, Err: %w", filePath, err)
+		}
+		err = writeFile(filePath, writeFlags, b)
+		if err != nil {
+			return 0, err
+		}
+	} else {
+		// This is a blob. Write it to the blobs folder.
+		algo := w.Opts.Desc.Digest.Algorithm()
+		blobDir := path.Join(w.Path, BlobsFolderName, algo.String())
+		if err := os.MkdirAll(blobDir, ContentFileMode); err != nil {
+			return 0, fmt.Errorf("error creating blobDir: %v, Err: %w", blobDir, err)
+		}
+		filePath = path.Join(blobDir, w.Opts.Desc.Digest.Encoded())
+
+		writeFlags, err := getWriteFlags(filePath)
+		if err != nil {
+			return 0, fmt.Errorf("error stat'ing file: %v, Err: %w", filePath, err)
+		}
+
+		// Now write the blob file.
+		err = writeFile(filePath, writeFlags, b)
+		if err != nil {
+			return 0, err
+		}
+	}
+	fInfo, err := os.Stat(filePath)
+	if err != nil {
+		return 0, fmt.Errorf("error retrieving FileInfo for file: %v, Err: %w", filePath, err)
+	}
+	w.Stat.UpdatedAt = fInfo.ModTime()
+	return len(b), nil
+}
+
+func (w *OciImageLayoutWriter) Close() error {
+	return nil
+}
+
+// Returns an empty digest until after Commit is called.
+func (w *OciImageLayoutWriter) Digest() digest.Digest {
+	return w.Dig
+}
+
+func (w *OciImageLayoutWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error {
+	w.Dig = w.Opts.Desc.Digest
+	return nil
+}
+
+func (w *OciImageLayoutWriter) Status() (content.Status, error) {
+	return w.Stat, nil
+}
+
+func (w *OciImageLayoutWriter) Truncate(size int64) error {
+	return errors.New("truncation is unsupported")
+}

oci/BUILD.bazel

@@ -38,6 +38,7 @@ bzl_library(
     visibility = ["//visibility:public"],
     deps = [
         ":image",
+        ":oci_image_layout",
         ":pull",
         ":push",
     ],
@@ -64,6 +65,16 @@ bzl_library(
     deps = ["@com_github_datadog_rules_oci//oci:providers"],
 )
 
+bzl_library(
+    name = "oci_image_layout",
+    srcs = ["oci_image_layout.bzl"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_datadog_rules_oci//oci:debug_flag",
+        "@com_github_datadog_rules_oci//oci:providers",
+    ],
+)
+
 bzl_library(
     name = "push",
     srcs = ["push.bzl"],

oci/defs.bzl

@@ -1,10 +1,12 @@
 load(":pull.bzl", _oci_pull = "oci_pull")
 load(":push.bzl", _oci_push = "oci_push")
 load(":image.bzl", _oci_image = "oci_image", _oci_image_index = "oci_image_index", _oci_image_layer = "oci_image_layer")
+load(":oci_image_layout.bzl", _oci_image_layout = "oci_image_layout")
 
 oci_pull = _oci_pull
 oci_push = _oci_push
 
 oci_image = _oci_image
 oci_image_index = _oci_image_index
 oci_image_layer = _oci_image_layer
+oci_image_layout = _oci_image_layout