Merge pull request #5343 from TomSweeneyRedHat/dev/tsweeney/picker2

[release-1.34] Cherry Pick a number of commits from main

Makefile

@@ -4,6 +4,10 @@ APPARMORTAG := $(shell hack/apparmor_tag.sh)
 STORAGETAGS := exclude_graphdriver_devicemapper $(shell ./btrfs_tag.sh) $(shell ./btrfs_installed_tag.sh) $(shell ./hack/libsubid_tag.sh)
 SECURITYTAGS ?= seccomp $(APPARMORTAG)
 TAGS ?= $(SECURITYTAGS) $(STORAGETAGS) $(shell ./hack/systemd_tag.sh)
+ifeq ($(shell uname -s),FreeBSD)
+# FreeBSD needs CNI until netavark is supported
+TAGS += cni
+endif
 BUILDTAGS += $(TAGS)
 PREFIX := /usr/local
 BINDIR := $(PREFIX)/bin
@@ -123,7 +127,7 @@ gopath:
 	test $(shell pwd) = $(shell cd ../../../../src/github.com/containers/buildah ; pwd)
 
 codespell:
-	codespell -S Makefile,buildah.spec.rpkg,AUTHORS,bin,vendor,.git,go.mod,go.sum,CHANGELOG.md,changelog.txt,seccomp.json,.cirrus.yml,"*.xz,*.gz,*.tar,*.tgz,*ico,*.png,*.1,*.5,*.orig,*.rej" -L passt,bu,uint,iff,od,erro -w
+	codespell -S Makefile,buildah.spec.rpkg,AUTHORS,bin,vendor,.git,go.mod,go.sum,CHANGELOG.md,changelog.txt,seccomp.json,.cirrus.yml,"*.xz,*.gz,*.tar,*.tgz,*ico,*.png,*.1,*.5,*.orig,*.rej" -L secon,passt,bu,uint,iff,od,erro -w
 
 .PHONY: validate
 validate: install.tools

add.go

@@ -23,7 +23,6 @@ import (
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/hashicorp/go-multierror"
 	digest "github.com/opencontainers/go-digest"
-	"github.com/opencontainers/runc/libcontainer/userns"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 )
@@ -438,7 +437,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 						ChmodDirs:     nil,
 						ChownFiles:    nil,
 						ChmodFiles:    nil,
-						IgnoreDevices: userns.RunningInUserNS(),
+						IgnoreDevices: runningInUserNS(),
 					}
 					putErr = copier.Put(extractDirectory, extractDirectory, putOptions, io.TeeReader(pipeReader, hasher))
 				}
@@ -579,7 +578,7 @@ func (b *Builder) Add(destination string, extract bool, options AddAndCopyOption
 						ChmodDirs:       nil,
 						ChownFiles:      nil,
 						ChmodFiles:      nil,
-						IgnoreDevices:   userns.RunningInUserNS(),
+						IgnoreDevices:   runningInUserNS(),
 					}
 					putErr = copier.Put(extractDirectory, extractDirectory, putOptions, io.TeeReader(pipeReader, hasher))
 				}

add_common.go

@@ -0,0 +1,8 @@
+//go:build !linux
+// +build !linux
+
+package buildah
+
+func runningInUserNS() bool {
+	return false
+}

add_linux.go

@@ -0,0 +1,9 @@
+package buildah
+
+import (
+	"github.com/opencontainers/runc/libcontainer/userns"
+)
+
+func runningInUserNS() bool {
+	return userns.RunningInUserNS()
+}

chroot/run_test.go

@@ -497,7 +497,7 @@ func TestMounts(t *testing.T) {
 					})
 				},
 				func(t *testing.T, report *types.TestReport) {
-					foundMounts := make(map[string]bool)
+					foundBindDestinationMount := false
 					for _, mount := range report.Spec.Mounts {
 						if mount.Destination == bind.destination {
 							allRequired := true
@@ -516,10 +516,12 @@ func TestMounts(t *testing.T) {
 									anyRejected = true
 								}
 							}
-							foundMounts[mount.Destination] = allRequired && !anyRejected
+							if allRequired && !anyRejected {
+								foundBindDestinationMount = true
+							}
 						}
 					}
-					if !foundMounts[bind.destination] {
+					if !foundBindDestinationMount {
 						t.Errorf("added mount for %s not found with the right flags (%v) in %+v", bind.destination, bind.options, report.Spec.Mounts)
 					}
 				},

cmd/buildah/commit.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/containers/buildah"
@@ -49,6 +50,7 @@ type commitInputOptions struct {
 	encryptionKeys     []string
 	encryptLayers      []int
 	unsetenvs          []string
+	addFile            []string
 }
 
 func init() {
@@ -77,6 +79,7 @@ func commitListFlagSet(cmd *cobra.Command, opts *commitInputOptions) {
 	flags := cmd.Flags()
 	flags.SetInterspersed(false)
 
+	flags.StringArrayVar(&opts.addFile, "add-file", nil, "add contents of a file to the image at a specified path (`source:destination`)")
 	flags.StringVar(&opts.authfile, "authfile", auth.GetDefaultAuthFile(), "path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override")
 	_ = cmd.RegisterFlagCompletionFunc("authfile", completion.AutocompleteDefault)
 	flags.StringVar(&opts.blobCache, "blob-cache", "", "assume image blobs in the specified directory will be available for pushing")
@@ -223,6 +226,28 @@ func commitCmd(c *cobra.Command, args []string, iopts commitInputOptions) error
 		}
 	}
 
+	var addFiles map[string]string
+	if len(iopts.addFile) > 0 {
+		addFiles = make(map[string]string)
+		for _, spec := range iopts.addFile {
+			specSlice := strings.SplitN(spec, ":", 2)
+			if len(specSlice) == 1 {
+				specSlice = []string{specSlice[0], specSlice[0]}
+			}
+			if len(specSlice) != 2 {
+				return fmt.Errorf("parsing add-file argument %q: expected 1 or 2 parts, got %d", spec, len(strings.SplitN(spec, ":", 2)))
+			}
+			st, err := os.Stat(specSlice[0])
+			if err != nil {
+				return fmt.Errorf("parsing add-file argument %q: source %q: %w", spec, specSlice[0], err)
+			}
+			if st.IsDir() {
+				return fmt.Errorf("parsing add-file argument %q: source %q is not a regular file", spec, specSlice[0])
+			}
+			addFiles[specSlice[1]] = specSlice[0]
+		}
+	}
+
 	options := buildah.CommitOptions{
 		PreferredManifestType: format,
 		Manifest:              iopts.manifest,
@@ -239,6 +264,7 @@ func commitCmd(c *cobra.Command, args []string, iopts commitInputOptions) error
 		UnsetEnvs:             iopts.unsetenvs,
 		OverrideChanges:       iopts.changes,
 		OverrideConfig:        overrideConfig,
+		ExtraImageContent:     addFiles,
 	}
 	exclusiveFlags := 0
 	if c.Flag("reference-time").Changed {

cmd/buildah/from.go

@@ -13,7 +13,6 @@ import (
 	"github.com/containers/buildah/pkg/cli"
 	"github.com/containers/buildah/pkg/parse"
 	"github.com/containers/common/pkg/auth"
-	"github.com/containers/common/pkg/config"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -188,11 +187,6 @@ func onBuild(builder *buildah.Builder, quiet bool) error {
 }
 
 func fromCmd(c *cobra.Command, args []string, iopts fromReply) error {
-	defaultContainerConfig, err := config.Default()
-	if err != nil {
-		return fmt.Errorf("failed to get container config: %w", err)
-	}
-
 	if len(args) == 0 {
 		return errors.New("an image name (or \"scratch\") must be specified")
 	}

cmd/buildah/main.go

@@ -59,8 +59,9 @@ var rootCmd = &cobra.Command{
 }
 
 var (
-	globalFlagResults globalFlags
-	exitCode          int
+	globalFlagResults      globalFlags
+	exitCode               int
+	defaultContainerConfig *config.Config
 )
 
 func init() {
@@ -79,12 +80,12 @@ func init() {
 		defaultStoreDriverOptions = optionSlice
 	}
 
-	containerConfig, err := config.Default()
+	defaultContainerConfig, err = config.Default()
 	if err != nil {
 		logrus.Errorf(err.Error())
 		os.Exit(1)
 	}
-	containerConfig.CheckCgroupsAndAdjustConfig()
+	defaultContainerConfig.CheckCgroupsAndAdjustConfig()
 
 	cobra.OnInitialize(initConfig)
 	// Disable the implicit `completion` command in cobra.
@@ -98,7 +99,7 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&globalFlagResults.UserShortNameAliasConfPath, "short-name-alias-conf", "", "path to short name alias cache file (not usually used)")
 	rootCmd.PersistentFlags().StringVar(&globalFlagResults.Root, "root", storageOptions.GraphRoot, "storage root dir")
 	rootCmd.PersistentFlags().StringVar(&globalFlagResults.RunRoot, "runroot", storageOptions.RunRoot, "storage state dir")
-	rootCmd.PersistentFlags().StringVar(&globalFlagResults.CgroupManager, "cgroup-manager", containerConfig.Engine.CgroupManager, "cgroup manager")
+	rootCmd.PersistentFlags().StringVar(&globalFlagResults.CgroupManager, "cgroup-manager", defaultContainerConfig.Engine.CgroupManager, "cgroup manager")
 	rootCmd.PersistentFlags().StringVar(&globalFlagResults.StorageDriver, "storage-driver", storageOptions.GraphDriverName, "storage-driver")
 	rootCmd.PersistentFlags().StringSliceVar(&globalFlagResults.StorageOpts, "storage-opt", defaultStoreDriverOptions, "storage driver option")
 	rootCmd.PersistentFlags().StringSliceVar(&globalFlagResults.UserNSUID, "userns-uid-map", []string{}, "default `ctrID:hostID:length` UID mapping to use")

cmd/buildah/manifest.go

@@ -231,7 +231,7 @@ func init() {
 	flags.StringVar(&manifestPushOpts.compressionFormat, "compression-format", "", "compression format to use")
 	flags.IntVar(&manifestPushOpts.compressionLevel, "compression-level", 0, "compression level to use")
 	flags.StringVarP(&manifestPushOpts.format, "format", "f", "", "manifest type (oci or v2s2) to attempt to use when pushing the manifest list (default is manifest type of source)")
-	flags.StringSliceVar(&manifestPushOpts.addCompression, "add-compression", nil, "add instances with selected compression while pushing")
+	flags.StringArrayVar(&manifestPushOpts.addCompression, "add-compression", defaultContainerConfig.Engine.AddCompression.Get(), "add instances with selected compression while pushing")
 	flags.BoolVarP(&manifestPushOpts.removeSignatures, "remove-signatures", "", false, "don't copy signatures when pushing images")
 	flags.StringVar(&manifestPushOpts.signBy, "sign-by", "", "sign the image using a GPG key with the specified `FINGERPRINT`")
 	flags.StringVar(&manifestPushOpts.signaturePolicy, "signature-policy", "", "`pathname` of signature policy file (not usually used)")

commit.go

@@ -118,6 +118,12 @@ type CommitOptions struct {
 	// to the configuration of the image that is being committed, after
 	// OverrideConfig is applied.
 	OverrideChanges []string
+	// ExtraImageContent is a map which describes additional content to add
+	// to the committed image.  The map's keys are filesystem paths in the
+	// image and the corresponding values are the paths of files whose
+	// contents will be used in their place.  The contents will be owned by
+	// 0:0 and have mode 0644.  Currently only accepts regular files.
+	ExtraImageContent map[string]string
 }
 
 var (

convertcw.go

@@ -171,6 +171,7 @@ func CWConvertImage(ctx context.Context, systemContext *types.SystemContext, sto
 		Slop:                     options.Slop,
 		FirmwareLibrary:          options.FirmwareLibrary,
 		Logger:                   logger,
+		GraphOptions:             store.GraphOptions(),
 	}
 	rc, workloadConfig, err := mkcw.Archive(sourceDir, &source.OCIv1, archiveOptions)
 	if err != nil {

convertcw_test.go

@@ -72,7 +72,7 @@ func TestCWConvertImage(t *testing.T) {
 	for _, status := range []int{http.StatusOK, http.StatusInternalServerError} {
 		for _, ignoreChainRetrievalErrors := range []bool{false, true} {
 			for _, ignoreAttestationErrors := range []bool{false, true} {
-				t.Run(fmt.Sprintf("status=%d,ignoreChainRetrievalErrors=%v,ignoreAttestationErrors=%v", status, ignoreChainRetrievalErrors, ignoreAttestationErrors), func(t *testing.T) {
+				t.Run(fmt.Sprintf("status~%d~ignoreChainRetrievalErrors~%v~ignoreAttestationErrors~%v", status, ignoreChainRetrievalErrors, ignoreAttestationErrors), func(t *testing.T) {
 					// create a per-test Store object
 					storeOptions := storage.StoreOptions{
 						GraphRoot:       t.TempDir(),

define/types.go

@@ -54,6 +54,9 @@ const (
 	SNP TeeType = "snp"
 )
 
+// DefaultRlimitValue is the value set by default for nofile and nproc
+const RLimitDefaultValue = uint64(1048576)
+
 // TeeType is a supported trusted execution environment type.
 type TeeType string
 
@@ -121,7 +124,7 @@ type ConfidentialWorkloadOptions struct {
 	AttestationURL           string
 	CPUs                     int
 	Memory                   int
-	TempDir                  string
+	TempDir                  string // used for the temporary plaintext copy of the disk image
 	TeeType                  TeeType
 	IgnoreAttestationErrors  bool
 	WorkloadID               string

docs/buildah-build.1.md

@@ -51,7 +51,7 @@ Set the ARCH of the image to be built, and that of the base image to be pulled,
 
 **--authfile** *path*
 
-Path of the authentication file. Default is ${XDG_RUNTIME_DIR}/containers/auth.json. If XDG_RUNTIME_DIR is not set, the default is /run/containers/$UID/auth.json. This file is created using `buildah login`.
+Path of the authentication file. Default is ${XDG_RUNTIME_DIR}/containers/auth.json. See containers-auth.json(5) for more information. This file is created using `buildah login`.
 
 If the authorization state is not found there, $HOME/.docker/config.json is checked, which is set using `docker login`.
 
@@ -708,6 +708,8 @@ Valid _type_ values are:
 If no type is specified, the value defaults to **local**.
 Alternatively, instead of a comma-separated sequence, the value of **--output** can be just a destination (in the `**dest** format) (e.g. `--output some-path`, `--output -`) where `--output some-path` is treated as if **type=local** and `--output -` is treated as if **type=tar**.
 
+Note: The **--tag** option can also be used to change the file image format to supported `containers-transports(5)`.
+
 **--pid** *how*
 
 Sets the configuration for PID namespaces when handling `RUN` instructions.
@@ -795,7 +797,7 @@ environment variable.  `export BUILDAH_RUNTIME=/usr/bin/crun`
 
 **--runtime-flag** *flag*
 
-Adds global flags for the container rutime. To list the supported flags, please
+Adds global flags for the container runtime. To list the supported flags, please
 consult the manpages of the selected container runtime.
 
 Note: Do not pass the leading `--` to the flag. To pass the runc flag `--log-format json`
@@ -872,6 +874,13 @@ Specifies the name which will be assigned to the resulting image if the build
 process completes successfully.
 If _imageName_ does not include a registry name component, the registry name *localhost* will be prepended to the image name.
 
+The **--tag** option supports all transports from `containers-transports(5)`.
+If no transport is specified, the `containers-storage` (i.e., local storage) transport is used.
+
+  __buildah build --tag=oci-archive:./foo.ociarchive .__
+
+  __buildah build -t quay.io/username/foo  .__
+
 **--target** *stageName*
 
 Set the target build stage to build.  When building a Containerfile with multiple build stages, --target
@@ -1022,12 +1031,11 @@ Set the architecture variant of the image to be pulled.
 
 Mount a host directory into containers when executing *RUN* instructions during
 the build.  The `OPTIONS` are a comma delimited list and can be:
-<sup>[[1]](#Footnote1)</sup>
 
    * [rw|ro]
    * [U]
    * [z|Z|O]
-   * [`[r]shared`|`[r]slave`|`[r]private`]
+   * [`[r]shared`|`[r]slave`|`[r]private`] <sup>[[1]](#Footnote1)</sup>
 
 The `CONTAINER-DIR` must be an absolute path such as `/src/docs`. The `HOST-DIR`
 must be an absolute path as well. Buildah bind-mounts the `HOST-DIR` to the
@@ -1310,7 +1318,7 @@ registries.conf is the configuration file which specifies which container regist
 Signature policy file.  This defines the trust policy for container images.  Controls which container registries can be used for image, and whether or not the tool should trust the images.
 
 ## SEE ALSO
-buildah(1), cpp(1), buildah-login(1), docker-login(1), namespaces(7), pid\_namespaces(7), containers-policy.json(5), containers-registries.conf(5), user\_namespaces(7), crun(1), runc(8), containers.conf(5), oci-hooks(5)
+buildah(1), cpp(1), buildah-login(1), docker-login(1), namespaces(7), pid\_namespaces(7), containers-policy.json(5), containers-registries.conf(5), user\_namespaces(7), crun(1), runc(8), containers.conf(5), oci-hooks(5), containers-transports(5), containers-auth.json(5)
 
 ## FOOTNOTES
 <a name="Footnote1">1</a>: The Buildah project is committed to inclusivity, a core value of open source. The `master` and `slave` mount propagation terminology used here is problematic and divisive, and should be changed. However, these terms are currently used within the Linux kernel and must be used as-is at this time. When the kernel maintainers rectify this usage, Buildah will follow suit immediately.

docs/buildah-commit.1.md

@@ -14,14 +14,24 @@ with a registry name component, `localhost` will be added to the name.  If
 name, the `buildah images` command will display `<none>` in the `REPOSITORY` and
 `TAG` columns.
 
+The *image* value supports all transports from `containers-transports(5)`. If no transport is specified, the `containers-storage` (i.e., local storage) transport is used.
+
 ## RETURN VALUE
 The image ID of the image that was created.  On error, 1 is returned and errno is returned.
 
 ## OPTIONS
 
+**--add-file** *source[:destination]*
+
+Read the contents of the file `source` and add it to the committed image as a
+file at `destination`.  If `destination` is not specified, the path of `source`
+will be used.  The new file will be owned by UID 0, GID 0, have 0644
+permissions, and be given a current timestamp unless the **--timestamp** option
+is also specified.  This option can be specified multiple times.
+
 **--authfile** *path*
 
-Path of the authentication file. Default is ${XDG_RUNTIME_DIR}/containers/auth.json. If XDG_RUNTIME_DIR is not set, the default is /run/containers/$UID/auth.json. This file is created using `buildah login`.
+Path of the authentication file. Default is ${XDG_RUNTIME_DIR}/containers/auth.json. See containers-auth.json(5) for more information. This file is created using `buildah login`.
 
 If the authorization state is not found there, $HOME/.docker/config.json is checked, which is set using `docker login`.
 
@@ -192,9 +202,12 @@ Unset environment variables from the final image.
 This example saves an image based on the container.
  `buildah commit containerID newImageName`
 
-This example saves an image named newImageName based on the container.
+This example saves an image named newImageName based on the container and removes the working container.
  `buildah commit --rm containerID newImageName`
 
+This example commits to an OCI archive file named /tmp/newImageName based on the container.
+ `buildah commit containerID oci-archive:/tmp/newImageName`
+
 This example saves an image with no name, removes the working container, and creates a new container using the image's ID.
  `buildah from $(buildah commit --rm containerID)`
 
@@ -260,4 +273,4 @@ registries.conf is the configuration file which specifies which container regist
 Signature policy file.  This defines the trust policy for container images.  Controls which container registries can be used for image, and whether or not the tool should trust the images.
 
 ## SEE ALSO
-buildah(1), buildah-images(1), containers-policy.json(5), containers-registries.conf(5)
+buildah(1), buildah-images(1), containers-policy.json(5), containers-registries.conf(5), containers-transports(5), containers-auth.json(5)