From eaa7b437e0a6a2d63295d419daf0d9ccab1239d2 Mon Sep 17 00:00:00 2001 From: Michael McCracken Date: Thu, 31 Oct 2024 20:52:49 -0700 Subject: [PATCH] Centralize metadata, move mount logic to atomfs pkg, add tests (#23) * write molecule config to metadata path The molecule config contains the OCI path and tag, which is important if we want to track what container image is mounted at a particular path, and thus what container might need to stop if a verity error is detected in one of the atoms in the molecule it's using. This commit writes that config to a JSON file in the metadata path. Signed-off-by: Michael McCracken * Move metadir to /run, mount logic into pkg 1 - To make it easy to read the config.json for a given mount, move the metadata path to /run/atomfs/meta/$current-mountns/$munged-mountpt-name/ using the current mount NS name in the path means we can track mounting different images on to the same mountpoint in different mount namespaces. 2 - move mount logic including the metadata dir logic from cmd/atomfs/mount.go to atomfs/molecule.go so that users of the package will also get the same metadir / config.json etc behavior that the `atomfs` tool does. As part of this move, we no longer mount an RO overlay in one place and then either remount another overlay or bind mount to the final target mountpoint. Instead we build one overlay mount for the mountpoint and either it has an upperdir/workdir or not. Signed-off-by: Michael McCracken * add flag to allow missing verity, enforce it pass through to molecule config, and check to be sure we don't guestmount and ignore verity data without explicitly saying we want to Signed-off-by: Michael McCracken * verify: return error when no squash mounts found In the guestmount case, we use squashfuse and there is no verity mount source to check. Treat this as a verify error. Signed-off-by: Michael McCracken * test: add bats tests for mount Add a bats tests suite for mounting and for failing to mount when the images are bad. Uses the ATOMFS_TEST_RUN_DIR env var to avoid polluting your host's /run/atomfs/meta dir. copies the guestmount test from the github yaml into bats and expands it a bit. I apologize for the bash quoting situation in the heredoc in the guestmount tests, forgive me Missing cases: - testing `atomfs verify` on bad images: requires manufacturing a verity image that will mount OK but has a bad block that won't get read until later. I have tested verify with mounted bad images that I mounted with a purposely broken atomfs, but there should be a better way for CI. Signed-off-by: Michael McCracken * move github test to use bats the existing test is now covered there, and we build our own test image so we can avoid the zothub dep and skopeo dep Signed-off-by: Michael McCracken * ensure workdir and upperdir are on same fs This redefines the --persist argument to point to a directory where atomfs will create or use both the workdir and the upperdir. So if you run `atomfs mount --persist=foo/` then the persistent writes will end up at foo/persist/. Signed-off-by: Michael McCracken * add metadir flag to substitute for /run/atomfs In some cases, e.g. when guestmounting in a new userns and mountns, but not chrooted, /run/atomfs may not be writable. In that situation, use the new --metadir flag to all commands to specify a replacement for the default /run/atomfs. This overlaps slightly with the ATOMFS_TEST_RUN_DIR environment variable, which would have the same effect, but should only be used for tests, as it is not discoverable. Signed-off-by: Michael McCracken --------- Signed-off-by: Michael McCracken --- .github/workflows/build.yaml | 16 +-- .gitignore | 3 + Makefile | 44 +++++++- README.md | 17 ++-- cmd/atomfs/mount.go | 131 +++++++----------------- cmd/atomfs/umount.go | 32 +++--- cmd/atomfs/utils.go | 22 ---- cmd/atomfs/verify.go | 37 +++++-- molecule.go | 189 ++++++++++++++++++++++++++++------- molecule_test.go | 2 +- oci.go | 19 +++- squashfs/squashfs.go | 3 +- squashfs/verity.go | 5 +- test/1.README.md | 2 + test/1.stacker.yaml | 14 +++ test/helpers.bash | 18 ++++ test/priv-mount.bats | 122 ++++++++++++++++++++++ test/priv-verify.bats | 36 +++++++ test/unpriv-guestmount.bats | 130 ++++++++++++++++++++++++ utils.go | 56 +++++++++++ 20 files changed, 690 insertions(+), 208 deletions(-) delete mode 100644 cmd/atomfs/utils.go create mode 100644 test/1.README.md create mode 100644 test/1.stacker.yaml create mode 100644 test/helpers.bash create mode 100644 test/priv-mount.bats create mode 100644 test/priv-verify.bats create mode 100644 test/unpriv-guestmount.bats create mode 100644 utils.go diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e380840..7b18cfc 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -51,12 +51,6 @@ jobs: echo "writing /etc/lxc/lxc-usernet" echo "$u veth lxcbr0 100" | sudo tee -a /etc/lxc/lxc-usernet - - name: install skopeo - run: | - mkdir ~/bin - wget -O ~/bin/skopeo --progress=dot:mega https://github.com/project-machine/tools/releases/download/v0.0.1/skopeo - chmod 755 ~/bin/skopeo - sudo cp -f ~/bin/skopeo /usr/bin/skopeo - name: lint run: | make gofmt @@ -71,15 +65,7 @@ jobs: cp ./bin/atomfs atomfs-${{ matrix.os }} - name: test run: | - export PATH=~/bin:$PATH - skopeo copy docker://zothub.io/machine/bootkit/bootkit:v0.0.16.230901-squashfs oci:oci:bootkit-squashfs - lxc-usernsexec -s << EOF - atomfs mount --persist=upper oci:bootkit-squashfs dest - [ -d dest/bootkit ] - touch dest/zz - atomfs umount dest - [ -f upper/zz ] - EOF + make batstest - name: Upload code coverage uses: codecov/codecov-action@v4 with: diff --git a/.gitignore b/.gitignore index d20591e..1b688a3 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,6 @@ VERSION # Output of the go coverage tool, specifically when used with LiteIDE *.out coverage.txt + +# bats test stuff +/bats-core diff --git a/Makefile b/Makefile index d896678..a9df465 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,14 @@ ROOT := $(shell git rev-parse --show-toplevel) GO_SRC_DIRS := $(shell find . -name "*.go" | xargs -n1 dirname | sort -u) GO_SRC := $(shell find . -name "*.go") VERSION_LDFLAGS=-X main.Version=$(MAIN_VERSION) +BATS = $(TOOLS_D)/bin/bats +BATS_VERSION := v1.10.0 +STACKER = $(TOOLS_D)/bin/stacker +STACKER_VERSION := v1.0.0 +TOOLS_D := $(ROOT)/tools + +export PATH := $(TOOLS_D)/bin:$(PATH) + .PHONY: gofmt gofmt: .made-gofmt @@ -23,6 +31,36 @@ atomfs: .made-gofmt $(GO_SRC) gotest: $(GO_SRC) go test -coverprofile=coverage.txt -ldflags "$(VERSION_LDFLAGS)" ./... -clean: - rm -f $(ROOT)/bin - rm .made-* +$(STACKER): + mkdir -p $(TOOLS_D)/bin + wget --progress=dot:giga https://github.com/project-stacker/stacker/releases/download/$(STACKER_VERSION)/stacker + chmod +x stacker + cp stacker $(TOOLS_D)/bin/ + +$(BATS): + mkdir -p $(TOOLS_D)/bin + git clone -b $(BATS_VERSION) https://github.com/bats-core/bats-core.git + cd bats-core; ./install.sh $(TOOLS_D) + mkdir -p $(ROOT)/test/test_helper + git clone --depth 1 https://github.com/bats-core/bats-support $(ROOT)/test/test_helper/bats-support + git clone --depth 1 https://github.com/bats-core/bats-assert $(ROOT)/test/test_helper/bats-assert + git clone --depth 1 https://github.com/bats-core/bats-file $(ROOT)/test/test_helper/bats-file + +batstest: $(BATS) $(STACKER) atomfs test/random.txt + cd $(ROOT)/test; sudo $(BATS) --tap --timing priv-*.bats + cd $(ROOT)/test; $(BATS) --tap --timing unpriv-*.bats + +test/random.txt: + dd if=/dev/random of=/dev/stdout count=2048 | base64 > test/random.txt + +.PHONY: test toolsclean +test: gotest batstest + +toolsclean: + rm -rf $(TOOLS_D) + rm -rf $(ROOT)/test/test_helper + rm -rf $(ROOT)/bats-core + +clean: toolsclean + rm -rf $(ROOT)/bin + rm -f .made-* diff --git a/README.md b/README.md index 88a4b7d..2d9646b 100644 --- a/README.md +++ b/README.md @@ -42,18 +42,19 @@ ab ## Implementation details -We create $mountpoint/meta and pass that to `atomfs` as the -Metadatapath. We do the readonly `atomfs` molecule mount -onto $metadir/ro. Then if a readonly mount is requested -$metadir/ro is bind mounted onto $metadir. Otherwise, we create -$metadir/work and $metadir/upper, and use these to do a rw -overlay mount of $metadir/ro onto $mountpoint. +The `atomfs` binary uses the `atomfs` package's Molecule API to mount oci +images. + +Each squashfs layer is mounted separately at a subdir under +`/run/atomfs/meta/$mountnsid/$mountpoint/`, and then an overlay mount is +constructed for the specified mountpath. If specified in the config, a writeable +upperdir is added to the overlay mount. Note that if you simply call `umount` on the mountpoint, then you will be left with all the individual squashfs mounts under -`dest/mounts/*/`. +`/run/atomfs/meta/$mountnsid/$mountpoint/`. Use `atomfs umount` instead. Note that you do need to be root in your namespace in order to -do the final bind or overlay mount. (We could get around this +do the final overlay mount. (We could get around this by using fuse-overlay, but creating a namespace seems overall tidy). diff --git a/cmd/atomfs/mount.go b/cmd/atomfs/mount.go index 22df8c2..4eec79d 100644 --- a/cmd/atomfs/mount.go +++ b/cmd/atomfs/mount.go @@ -1,16 +1,15 @@ package main import ( - "errors" "fmt" "os" "os/exec" "path/filepath" "strings" - "syscall" + "github.com/pkg/errors" "github.com/urfave/cli" - "golang.org/x/sys/unix" + "machinerun.io/atomfs" "machinerun.io/atomfs/squashfs" ) @@ -22,13 +21,21 @@ var mountCmd = cli.Command{ Action: doMount, Flags: []cli.Flag{ cli.StringFlag{ - Name: "persist, upper, upperdir", - Usage: "Specify a directory to use as writeable overlay (implies --writeable)", + Name: "persist", + Usage: "Specify a directory to use for the workdir and upperdir of a writeable overlay (implies --writeable)", }, cli.BoolFlag{ Name: "writeable, writable", Usage: "Make the mount writeable using an overlay (ephemeral by default)", }, + cli.BoolFlag{ + Name: "allow-missing-verity", + Usage: "Mount even if the image has no verity data", + }, + cli.StringFlag{ + Name: "metadir", + Usage: "Directory to use for metadata. Use this if /run/atomfs is not writable for some reason.", + }, }, } @@ -44,7 +51,7 @@ func findImage(ctx *cli.Context) (string, string, error) { } ocidir := r[0] tag := r[1] - if !PathExists(ocidir) { + if !atomfs.PathExists(ocidir) { return "", "", fmt.Errorf("oci directory %s does not exist: %w", ocidir, mountUsage(ctx.App.Name)) } return ocidir, tag, nil @@ -70,92 +77,44 @@ func doMount(ctx *cli.Context) error { os.Exit(1) } target := ctx.Args()[1] - metadir := filepath.Join(target, "meta") - - complete := false - - defer func() { - if !complete { - cleanupDest(metadir) - } - }() - - if PathExists(metadir) { - return fmt.Errorf("%q exists: cowardly refusing to mess with it", metadir) - } - - if err := EnsureDir(metadir); err != nil { - return err - } - - rodest := filepath.Join(metadir, "ro") - if err = EnsureDir(rodest); err != nil { - return err - } - - opts := atomfs.MountOCIOpts{ - OCIDir: ocidir, - MetadataPath: metadir, - Tag: tag, - Target: rodest, - } - - mol, err := atomfs.BuildMoleculeFromOCI(opts) + absTarget, err := filepath.Abs(target) if err != nil { return err } - err = mol.Mount(rodest) + absOCIDir, err := filepath.Abs(ocidir) if err != nil { return err } - if ctx.Bool("writeable") || ctx.IsSet("persist") { - err = overlay(target, rodest, metadir, ctx) - } else { - err = bind(target, rodest) - } - - complete = err == nil - return err -} - -func cleanupDest(metadir string) { - fmt.Printf("Failure detected: cleaning up %q", metadir) - rodest := filepath.Join(metadir, "ro") - if PathExists(rodest) { - if err := unix.Unmount(rodest, 0); err != nil { - fmt.Printf("Failed unmounting %q: %v", rodest, err) + persistPath := "" + if ctx.IsSet("persist") { + persistPath = ctx.String("persist") + if persistPath == "" { + return fmt.Errorf("--persist requires an argument") } } + opts := atomfs.MountOCIOpts{ + OCIDir: absOCIDir, + Tag: tag, + Target: absTarget, + AddWriteableOverlay: ctx.Bool("writeable") || ctx.IsSet("persist"), + WriteableOverlayPath: persistPath, + AllowMissingVerityData: ctx.Bool("allow-missing-verity"), + MetadataDir: ctx.String("metadir"), // nil here means /run/atomfs + } - mountsdir := filepath.Join(metadir, "mounts") - entries, err := os.ReadDir(mountsdir) + mol, err := atomfs.BuildMoleculeFromOCI(opts) if err != nil { - fmt.Printf("Failed reading contents of %q: %v", mountsdir, err) - os.RemoveAll(metadir) - return + return errors.Wrapf(err, "couldn't build molecule with opts %+v", opts) } - wd, err := os.Getwd() + err = mol.Mount(target) if err != nil { - fmt.Printf("Failed getting working directory") - os.RemoveAll(metadir) + return errors.Wrapf(err, "couldn't mount molecule at mntpt %q ", target) } - for _, e := range entries { - n := filepath.Base(e.Name()) - if n == "workaround" { - continue - } - if strings.HasSuffix(n, ".log") { - continue - } - p := filepath.Join(wd, mountsdir, e.Name()) - if err := squashUmount(p); err != nil { - fmt.Printf("Failed unmounting %q: %v\n", p, err) - } - } - os.RemoveAll(metadir) + + return nil } func RunCommand(args ...string) error { @@ -177,23 +136,3 @@ func squashUmount(p string) error { } return RunCommand("fusermount", "-u", p) } - -func overlay(target, rodest, metadir string, ctx *cli.Context) error { - workdir := filepath.Join(metadir, "work") - if err := EnsureDir(workdir); err != nil { - return err - } - upperdir := filepath.Join(metadir, "persist") - if ctx.IsSet("persist") { - upperdir = ctx.String("persist") - } - if err := EnsureDir(upperdir); err != nil { - return err - } - overlayArgs := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s,index=off,userxattr", rodest, upperdir, workdir) - return unix.Mount("overlayfs", target, "overlay", 0, overlayArgs) -} - -func bind(target, source string) error { - return syscall.Mount(source, target, "", syscall.MS_BIND, "") -} diff --git a/cmd/atomfs/umount.go b/cmd/atomfs/umount.go index a889f09..fde7340 100644 --- a/cmd/atomfs/umount.go +++ b/cmd/atomfs/umount.go @@ -7,6 +7,7 @@ import ( "syscall" "github.com/urfave/cli" + "machinerun.io/atomfs" "machinerun.io/atomfs/mount" ) @@ -15,6 +16,12 @@ var umountCmd = cli.Command{ Usage: "unmount atomfs image", ArgsUsage: "mountpoint", Action: doUmount, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "metadir", + Usage: "Directory to use for metadata. Use this if /run/atomfs is not writable for some reason.", + }, + }, } func umountUsage(me string) error { @@ -43,36 +50,37 @@ func doUmount(ctx *cli.Context) error { } } - // We expect the argument to be the mountpoint - either a readonly - // bind mount, or a writeable overlay. + // We expect the argument to be the mountpoint of the overlay err = syscall.Unmount(mountpoint, 0) if err != nil { errs = append(errs, fmt.Errorf("Failed unmounting %s: %v", mountpoint, err)) } - // Now that we've unmounted the mountpoint, we expect the following - // under there: - // $mountpoint/meta/ro - the original readonly overlay mountpoint - // $mountpoint/meta/mounts/* - the original squashfs mounts - metadir := filepath.Join(mountpoint, "meta") - p := filepath.Join(metadir, "ro") - err = syscall.Unmount(p, 0) + // We expect the following in the metadir + // + // $metadir/mounts/* - the original squashfs mounts + // $metadir/meta/config.json + + // TODO: want to know mountnsname for a target mountpoint... not for our current proc??? + mountNSName, err := atomfs.GetMountNSName() if err != nil { - errs = append(errs, fmt.Errorf("Failed unmounting RO mountpoint %s: %v", p, err)) + errs = append(errs, fmt.Errorf("Failed to get mount namespace name")) } + metadir := filepath.Join(atomfs.RuntimeDir(ctx.String("metadir")), "meta", mountNSName, atomfs.ReplacePathSeparators(mountpoint)) mountsdir := filepath.Join(metadir, "mounts") mounts, err := os.ReadDir(mountsdir) if err != nil { errs = append(errs, fmt.Errorf("Failed reading list of mounts: %v", err)) - return fmt.Errorf("Encountered errors: %#v", errs) + return fmt.Errorf("Encountered errors: %v", errs) } for _, m := range mounts { - p = filepath.Join(mountsdir, m.Name()) + p := filepath.Join(mountsdir, m.Name()) if !m.IsDir() || !isMountpoint(p) { continue } + err = syscall.Unmount(p, 0) if err != nil { errs = append(errs, fmt.Errorf("Failed unmounting squashfs dir %s: %v", p, err)) diff --git a/cmd/atomfs/utils.go b/cmd/atomfs/utils.go deleted file mode 100644 index dc811fb..0000000 --- a/cmd/atomfs/utils.go +++ /dev/null @@ -1,22 +0,0 @@ -package main - -import ( - "fmt" - "os" -) - -func EnsureDir(dir string) error { - err := os.MkdirAll(dir, 0755) - if err != nil { - return fmt.Errorf("Failed creating directory %s: %w", dir, err) - } - return nil -} - -func PathExists(d string) bool { - _, err := os.Stat(d) - if err != nil && os.IsNotExist(err) { - return false - } - return true -} diff --git a/cmd/atomfs/verify.go b/cmd/atomfs/verify.go index a0057b2..c587405 100644 --- a/cmd/atomfs/verify.go +++ b/cmd/atomfs/verify.go @@ -6,6 +6,8 @@ import ( "strings" "github.com/urfave/cli" + "machinerun.io/atomfs" + "machinerun.io/atomfs/log" "machinerun.io/atomfs/mount" "machinerun.io/atomfs/squashfs" ) @@ -15,6 +17,12 @@ var verifyCmd = cli.Command{ Usage: "check atomfs image for dm-verity errors", ArgsUsage: "atomfs mountpoint", Action: doVerify, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "metadir", + Usage: "Directory to use for metadata. Use this if /run/atomfs is not writable for some reason.", + }, + }, } func verifyUsage(me string) error { @@ -41,9 +49,13 @@ func doVerify(ctx *cli.Context) error { return fmt.Errorf("%s is not a mountpoint", mountpoint) } - // hidden by the final overlay mount, but visible in the mountinfo: - // $mountpoint/meta/mounts/* - the original squashfs mounts - mountsdir := filepath.Join(mountpoint, "meta", "mounts") + mountNSName, err := atomfs.GetMountNSName() + if err != nil { + return err + } + + metadir := filepath.Join(atomfs.RuntimeDir(ctx.String("metadir")), "meta", mountNSName, atomfs.ReplacePathSeparators(mountpoint)) + mountsdir := filepath.Join(metadir, "mounts") mounts, err := mount.ParseMounts("/proc/self/mountinfo") if err != nil { @@ -58,16 +70,19 @@ func doVerify(ctx *cli.Context) error { } allOK := true + checkedCount := 0 for _, m := range mounts { - - if m.FSType != "squashfs" { + if !strings.HasPrefix(m.Target, mountsdir) { continue } - - if !strings.HasPrefix(m.Target, mountsdir) { + if m.FSType == "fuse.squashfuse_ll" { + log.Warnf("found squashfuse mount not supported by verify at %q", m.Source) continue } - + if m.FSType != "squashfs" { + continue + } + checkedCount = checkedCount + 1 err = squashfs.ConfirmExistingVerityDeviceCurrentValidity(m.Source) if err != nil { fmt.Printf("%s: CORRUPTION FOUND\n", m.Source) @@ -77,6 +92,12 @@ func doVerify(ctx *cli.Context) error { } } + // TODO - want to also be able to compare to expected # of mounts from + // molecule, need to write more molecule info during mol.mount + if checkedCount == 0 { + return fmt.Errorf("no applicable mounts found in %q", mountsdir) + } + if allOK { return nil } diff --git a/molecule.go b/molecule.go index 619bec5..f22e0c9 100644 --- a/molecule.go +++ b/molecule.go @@ -1,6 +1,7 @@ package atomfs import ( + "fmt" "os" "path" "path/filepath" @@ -9,6 +10,7 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/sys/unix" + "machinerun.io/atomfs/log" "machinerun.io/atomfs/mount" "machinerun.io/atomfs/squashfs" ) @@ -21,34 +23,67 @@ type Molecule struct { config MountOCIOpts } +func (m Molecule) MetadataPath() (string, error) { + + mountNSName, err := GetMountNSName() + if err != nil { + return "", err + } + absTarget, err := filepath.Abs(m.config.Target) + if err != nil { + return "", err + } + metadir := filepath.Join(RuntimeDir(m.config.MetadataDir), "meta", mountNSName, ReplacePathSeparators(absTarget)) + return metadir, nil +} + +func (m Molecule) MountedAtomsPath(parts ...string) (string, error) { + metapath, err := m.MetadataPath() + if err != nil { + return "", err + } + mounts := path.Join(metapath, "mounts") + return path.Join(append([]string{mounts}, parts...)...), nil +} + // mountUnderlyingAtoms mounts all the underlying atoms at -// config.MountedAtomsPath(). -func (m Molecule) mountUnderlyingAtoms() error { +// MountedAtomsPath(). +// it returns a cleanup function that will tear down any atoms it successfully mounted. +func (m Molecule) mountUnderlyingAtoms() (error, func()) { // in the case that we have a verity or other mount error we need to // tear down the other underlying atoms so we don't leave verity and loop // devices around unused. atomsMounted := []string{} - cleanupAtoms := func(err error) error { + cleanupAtoms := func() { for _, target := range atomsMounted { if umountErr := squashfs.Umount(target); umountErr != nil { - return errors.Wrapf(umountErr, "failed to unmount atom @ target %q while handling error: %s", target, err) + log.Warnf("cleanup: failed to unmount atom @ target %q: %s", target, umountErr) } } - return err } + noop := func() {} for _, a := range m.Atoms { - target := m.config.MountedAtomsPath(a.Digest.Encoded()) + target, err := m.MountedAtomsPath(a.Digest.Encoded()) + if err != nil { + return errors.Wrapf(err, "failed to find mounted atoms path for %+v", a), cleanupAtoms + } rootHash := a.Annotations[squashfs.VerityRootHashAnnotation] - if !m.config.AllowMissingVerityData && rootHash == "" { - return errors.Errorf("%v is missing verity data", a.Digest) + if !m.config.AllowMissingVerityData { + + if rootHash == "" { + return errors.Errorf("%v is missing verity data", a.Digest), cleanupAtoms + } + if !squashfs.AmHostRoot() { + return errors.Errorf("won't guestmount an image with verity data without --allow-missing-verity"), cleanupAtoms + } } mounts, err := mount.ParseMounts("/proc/self/mountinfo") if err != nil { - return err + return err, cleanupAtoms } mountpoint, mounted := mounts.FindMount(target) @@ -59,38 +94,40 @@ func (m Molecule) mountUnderlyingAtoms() error { rootHash, m.config.AllowMissingVerityData) if err != nil { - return err + return err, cleanupAtoms } err = squashfs.ConfirmExistingVerityDeviceCurrentValidity(mountpoint.Source) if err != nil { - return err + return err, cleanupAtoms } } continue } if err := os.MkdirAll(target, 0755); err != nil { - return err + return err, cleanupAtoms } err = squashfs.Mount(m.config.AtomsPath(a.Digest.Encoded()), target, rootHash) if err != nil { - return cleanupAtoms(err) + return err, cleanupAtoms } atomsMounted = append(atomsMounted, target) } - return nil + return nil, noop } -// overlayArgs - returns all of the mount options to pass to the kernel to -// actually mount this molecule. -// This function assumes read-only. It does not provide upperdir or workdir. -func (m Molecule) overlayArgs(dest string) (string, error) { +// overlayArgs - returns a colon-separated string of dirs to be used as mount +// options to pass to the kernel to actually mount this molecule. +func (m Molecule) overlayLowerDirs() (string, error) { dirs := []string{} for _, a := range m.Atoms { - target := m.config.MountedAtomsPath(a.Digest.Encoded()) + target, err := m.MountedAtomsPath(a.Digest.Encoded()) + if err != nil { + return "", err + } dirs = append(dirs, target) } @@ -99,7 +136,10 @@ func (m Molecule) overlayArgs(dest string) (string, error) { // We create an empty directory called "workaround" in the mounts // directory, and add that to lowerdir list. if len(dirs) == 1 { - workaround := m.config.MountedAtomsPath("workaround") + workaround, err := m.MountedAtomsPath("workaround") + if err != nil { + return "", err + } if err := os.MkdirAll(workaround, 0755); err != nil { return "", errors.Wrapf(err, "couldn't make workaround dir") } @@ -109,8 +149,8 @@ func (m Molecule) overlayArgs(dest string) (string, error) { // Note that in overlayfs, the first thing is the top most layer in the // overlay. - mntOpts := "index=off,xino=on,userxattr,lowerdir=" + strings.Join(dirs, ":") - return mntOpts, nil + + return strings.Join(dirs, ":"), nil } // device mapper has no namespacing. if two different binaries invoke this code @@ -119,14 +159,14 @@ func (m Molecule) overlayArgs(dest string) (string, error) { // device exists. so try to cooperate via this lock. var advisoryLockPath = path.Join(os.TempDir(), ".atomfs-lock") -func makeLock(mountpoint string) (*os.File, error) { +func makeLock(lockdir string) (*os.File, error) { lockfile, err := os.Create(advisoryLockPath) if err == nil { return lockfile, nil } // backup plan: lock the destination as ${path}.atomfs-lock - mountpoint = strings.TrimSuffix(mountpoint, "/") - lockPath := filepath.Join(mountpoint, ".atomfs-lock") + lockdir = strings.TrimSuffix(lockdir, "/") + lockPath := filepath.Join(lockdir, ".atomfs-lock") var err2 error lockfile, err2 = os.Create(lockPath) if err2 == nil { @@ -137,8 +177,24 @@ func makeLock(mountpoint string) (*os.File, error) { return lockfile, err } +var OverlayMountOptions = "index=off,xino=on,userxattr" + +// Mount mounts an overlay at dest, with writeable overlay as per m.config func (m Molecule) Mount(dest string) error { - lockfile, err := makeLock(dest) + + metadir, err := m.MetadataPath() + if err != nil { + return errors.Wrapf(err, "can't find metadata path") + } + if PathExists(metadir) { + return fmt.Errorf("%q exists: cowardly refusing to mess with it", metadir) + } + + if err := EnsureDir(metadir); err != nil { + return err + } + + lockfile, err := makeLock(metadir) if err != nil { return errors.WithStack(err) } @@ -149,25 +205,86 @@ func (m Molecule) Mount(dest string) error { return errors.WithStack(err) } - mntOpts, err := m.overlayArgs(dest) + overlayLowerDirs, err := m.overlayLowerDirs() if err != nil { return err } - // The kernel doesn't allow mount options longer than 4096 chars, so - // let's give a nicer error than -EINVAL here. - if len(mntOpts) > 4096 { - return errors.Errorf("too many lower dirs; must have fewer than 4096 chars") + complete := false + + defer func() { + if !complete { + log.Errorf("Failure detected: cleaning up %q", metadir) + os.RemoveAll(metadir) + } + }() + + err, cleanupUnderlyingAtoms := m.mountUnderlyingAtoms() + if err != nil { + return err } - err = m.mountUnderlyingAtoms() + defer func() { + if !complete { + cleanupUnderlyingAtoms() + } + }() + + err = m.config.WriteToFile(filepath.Join(metadir, "config.json")) if err != nil { return err } - // now, do the actual overlay mount - err = unix.Mount("overlay", dest, "overlay", 0, mntOpts) - return errors.Wrapf(err, "couldn't do overlay mount to %s, opts: %s", dest, mntOpts) + overlayArgs := "" + if m.config.AddWriteableOverlay { + rodest := filepath.Join(metadir, "ro") + if err = EnsureDir(rodest); err != nil { + return err + } + + persistMetaPath := m.config.WriteableOverlayPath + if persistMetaPath == "" { + // no configured path, use metadir + persistMetaPath = metadir + } + + workdir := filepath.Join(persistMetaPath, "work") + if err := EnsureDir(workdir); err != nil { + return errors.Wrapf(err, "failed to ensure workdir %q", workdir) + } + + upperdir := filepath.Join(persistMetaPath, "persist") + if err := EnsureDir(upperdir); err != nil { + return errors.Wrapf(err, "failed to ensure upperdir %q", upperdir) + } + + defer func() { + if !complete && m.config.WriteableOverlayPath == "" { + os.RemoveAll(m.config.WriteableOverlayPath) + } + }() + + overlayArgs = fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s,%s", dest, overlayLowerDirs, upperdir, workdir, OverlayMountOptions) + + } else { + // for readonly, just mount the overlay directly onto dest + overlayArgs = fmt.Sprintf("lowerdir=%s,%s", overlayLowerDirs, OverlayMountOptions) + + } + + // The kernel doesn't allow mount options longer than 4096 chars + if len(overlayArgs) > 4096 { + return errors.Errorf("too many lower dirs; must have fewer than 4096 chars") + } + + err = unix.Mount("overlay", dest, "overlay", 0, overlayArgs) + if err != nil { + return errors.Wrapf(err, "couldn't do overlay mount to %s, opts: %s", dest, overlayArgs) + } + + // ensure deferred cleanups become noops: + complete = true + return nil } func Umount(dest string) error { @@ -199,7 +316,7 @@ func Umount(dest string) error { continue } - if m.Target != dest { + if m.Target != dest { // TODO is this still right continue } diff --git a/molecule_test.go b/molecule_test.go index 758db0e..973aeea 100644 --- a/molecule_test.go +++ b/molecule_test.go @@ -20,7 +20,7 @@ func TestAllowMissingVerityData(t *testing.T) { Atoms: []ispec.Descriptor{{Digest: d}}, } - err := mol.mountUnderlyingAtoms() + err, _ := mol.mountUnderlyingAtoms() assert.NotNil(err) assert.Equal(fmt.Sprintf("sha256:%s is missing verity data", hash), err.Error()) } diff --git a/oci.go b/oci.go index 2e9fc15..500247e 100644 --- a/oci.go +++ b/oci.go @@ -1,6 +1,8 @@ package atomfs import ( + "encoding/json" + "io/ioutil" "path" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -10,10 +12,12 @@ import ( type MountOCIOpts struct { OCIDir string - MetadataPath string Tag string Target string + AddWriteableOverlay bool + WriteableOverlayPath string AllowMissingVerityData bool + MetadataDir string } func (c MountOCIOpts) AtomsPath(parts ...string) string { @@ -21,9 +25,16 @@ func (c MountOCIOpts) AtomsPath(parts ...string) string { return path.Join(append([]string{atoms}, parts...)...) } -func (c MountOCIOpts) MountedAtomsPath(parts ...string) string { - mounts := path.Join(c.MetadataPath, "mounts") - return path.Join(append([]string{mounts}, parts...)...) +func (c MountOCIOpts) WriteToFile(filename string) error { + b, err := json.Marshal(c) + if err != nil { + return err + } + err = ioutil.WriteFile(filename, b, 0644) + if err != nil { + return err + } + return nil } func BuildMoleculeFromOCI(opts MountOCIOpts) (Molecule, error) { diff --git a/squashfs/squashfs.go b/squashfs/squashfs.go index 328b061..3b26b5c 100644 --- a/squashfs/squashfs.go +++ b/squashfs/squashfs.go @@ -18,6 +18,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/pkg/errors" "golang.org/x/sys/unix" + "machinerun.io/atomfs/log" "machinerun.io/atomfs/mount" ) @@ -499,7 +500,7 @@ func (k *KernelExtractor) Name() string { } func (k *KernelExtractor) IsAvailable() error { - if !amHostRoot() { + if !AmHostRoot() { return errors.Errorf("not host root") } return nil diff --git a/squashfs/verity.go b/squashfs/verity.go index c274772..ea1bf7b 100644 --- a/squashfs/verity.go +++ b/squashfs/verity.go @@ -87,6 +87,7 @@ import ( "github.com/martinjungblut/go-cryptsetup" "github.com/pkg/errors" "golang.org/x/sys/unix" + "machinerun.io/atomfs/mount" ) @@ -266,7 +267,7 @@ func uidmapIsHost(oneline string) bool { return true } -func amHostRoot() bool { +func AmHostRoot() bool { // if not uid 0, not host root if os.Geteuid() != 0 { return false @@ -280,7 +281,7 @@ func amHostRoot() bool { } func Mount(squashfs, mountpoint, rootHash string) error { - if !amHostRoot() { + if !AmHostRoot() { return GuestMount(squashfs, mountpoint) } err := HostMount(squashfs, mountpoint, rootHash) diff --git a/test/1.README.md b/test/1.README.md new file mode 100644 index 0000000..32fc2ca --- /dev/null +++ b/test/1.README.md @@ -0,0 +1,2 @@ +# Just a file to import into a scratch stacker image + diff --git a/test/1.stacker.yaml b/test/1.stacker.yaml new file mode 100644 index 0000000..5f03389 --- /dev/null +++ b/test/1.stacker.yaml @@ -0,0 +1,14 @@ +test_base: + from: + type: scratch + imports: + - path: 1.README.md + dest: / + +test: + from: + type: built + tag: test_base + imports: + - path: random.txt + dest: / diff --git a/test/helpers.bash b/test/helpers.bash new file mode 100644 index 0000000..e8599c6 --- /dev/null +++ b/test/helpers.bash @@ -0,0 +1,18 @@ + +check_root(){ + if [ "$(id -u)" != "0" ]; then + echo "you should be root to run this suite" + exit 1 + fi +} + +ROOT_D=$(dirname $BATS_TEST_FILENAME)/.. +TOOLS_D=$ROOT_D/tools +export PATH="$TOOLS_D/bin:$ROOT_D/bin:$PATH" + +build_image_at() { + cd $1 + sudo env "PATH=$PATH" stacker --oci-dir $1/oci --stacker-dir=$1/stacker --roots-dir=$1/roots --debug build -f $(dirname $BATS_TEST_FILENAME)/1.stacker.yaml --layer-type squashfs + sudo env "PATH=$PATH" stacker --oci-dir $1/oci-no-verity --stacker-dir=$1/stacker --roots-dir=$1/roots --debug build -f $(dirname $BATS_TEST_FILENAME)/1.stacker.yaml --layer-type squashfs --no-squashfs-verity + sudo chown -R $(id -un):$(id -gn) $1/oci $1/oci-no-verity $1/stacker $1/roots +} diff --git a/test/priv-mount.bats b/test/priv-mount.bats new file mode 100644 index 0000000..dc8c4f8 --- /dev/null +++ b/test/priv-mount.bats @@ -0,0 +1,122 @@ +load helpers +load 'test_helper/bats-support/load' +load 'test_helper/bats-assert/load' +load 'test_helper/bats-file/load' + +function setup_file() { + check_root + build_image_at $BATS_SUITE_TMPDIR + export ATOMFS_TEST_RUN_DIR=${BATS_SUITE_TMPDIR}/run/atomfs + mkdir -p $ATOMFS_TEST_RUN_DIR + export MY_MNTNSNAME=$(readlink /proc/self/ns/mnt | cut -c 6-15) +} + +function setup() { + export MP=${BATS_TEST_TMPDIR}/testmountpoint + mkdir -p $MP +} + +@test "RO mount/umount and verify of good image works" { + run atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP + assert_success + assert_file_exists $MP/1.README.md + assert_file_exists $MP/random.txt + assert_dir_exists $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/ + + run touch $MP/do-not-let-me + assert_failure + + run atomfs verify $MP + assert_success + + run atomfs --debug umount $MP + assert_success + + # mount point and meta dir should exist but be empty: + assert_dir_exists $MP + assert [ -z $( ls -A $MP) ] + assert_dir_exists $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/ + assert [ -z $( ls -A $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/ ) ] + +} + +@test "mount with missing verity data fails" { + run atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP + assert_failure + assert_line --partial "is missing verity data" + + # mount point and meta dir should exist but be empty: + assert_dir_exists $MP + assert [ -z $( ls -A $MP) ] + assert_dir_exists $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/ + assert [ -z $( ls -A $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/ ) ] + +} + +@test "mount with missing verity data passes if you ignore it" { + run atomfs --debug mount --allow-missing-verity ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP + assert_success + + run atomfs --debug umount $MP + assert_success + + # mount point and meta dir should exist but be empty: + assert_dir_exists $MP + assert [ -z $( ls -A $MP) ] + assert_dir_exists $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/ + assert [ -z $( ls -A $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/ ) ] + +} + +@test "mount/umount with writeable overlay" { + run atomfs --debug mount --writeable ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP + assert_success + assert_file_exists $MP/1.README.md + assert_file_exists $MP/random.txt + assert_dir_exists $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/ + + run touch $MP/this-time-let-me + assert_success + + run cp $MP/1.README.md $MP/3.README.md + assert_success + + run atomfs --debug umount $MP + assert_success + + # mount point and meta dir should exist but be empty: + assert_dir_exists $MP + assert [ -z $( ls -A $MP) ] + assert_dir_exists $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/ + assert [ -z $( ls -A $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/ ) ] +} + +@test "mount with writeable overlay in separate dir" { + export PERSIST_DIR=${BATS_TEST_TMPDIR}/persist-dir + mkdir -p $PERSIST_DIR + run atomfs --debug mount --persist=${PERSIST_DIR} ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP + assert_success + assert_file_exists $MP/1.README.md + assert_file_exists $MP/random.txt + + run touch $MP/this-time-let-me + assert_success + run cp $MP/1.README.md $MP/3.README.md + assert_success + + assert_file_exists $PERSIST_DIR/persist/this-time-let-me + assert_file_exists $PERSIST_DIR/persist/3.README.md + assert_file_not_exists $PERSIST_DIR/persist/1.README.md + + run atomfs --debug umount $MP + assert_success + # mount point and meta dir should exist but be empty: + assert_dir_exists $MP + assert [ -z $( ls -A $MP) ] + assert_dir_exists $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/ + assert [ -z $( ls -A $ATOMFS_TEST_RUN_DIR/meta/$MY_MNTNSNAME/) ] + + # but persist dir should still be there: + assert_file_exists $PERSIST_DIR/persist/this-time-let-me + assert_file_exists $PERSIST_DIR/persist/3.README.md +} diff --git a/test/priv-verify.bats b/test/priv-verify.bats new file mode 100644 index 0000000..5ab78e5 --- /dev/null +++ b/test/priv-verify.bats @@ -0,0 +1,36 @@ +load helpers +load 'test_helper/bats-support/load' +load 'test_helper/bats-assert/load' +load 'test_helper/bats-file/load' + +function setup_file() { + export ATOMFS_TEST_RUN_DIR=${BATS_SUITE_TMPDIR}/run/atomfs + mkdir -p $ATOMFS_TEST_RUN_DIR +} + +@test "mounting tampered small images fails immediately" { + build_image_at $BATS_TEST_TMPDIR + + sha256sum $BATS_TEST_TMPDIR/oci/blobs/sha256/* > initialsums + + # write some bad data onto the squash blobs to make them invalid + for blob in $BATS_TEST_TMPDIR/oci/blobs/sha256/* ; do + file $blob | grep "Squashfs filesystem" || continue + dd if=/dev/random of=$blob conv=notrunc seek=100 count=100 + done + + sha256sum $BATS_TEST_TMPDIR/oci/blobs/sha256/* > finalsums + + # the sums should be different, so assert that diff finds diffs: + run diff initialsums finalsums + assert_failure + + mkdir -p mountpoint + run atomfs --debug mount ${BATS_TEST_TMPDIR}/oci:test-squashfs mountpoint + assert_failure + +} + +@test "TODO: check atomfs verify on a mounted image that isn't detected immediately" { + echo TODO +} diff --git a/test/unpriv-guestmount.bats b/test/unpriv-guestmount.bats new file mode 100644 index 0000000..a2db887 --- /dev/null +++ b/test/unpriv-guestmount.bats @@ -0,0 +1,130 @@ +load helpers +load test_helper/bats-support/load +load test_helper/bats-assert/load +load test_helper/bats-file/load + +function setup_file() { + build_image_at $BATS_SUITE_TMPDIR + export ATOMFS_TEST_RUN_DIR=${BATS_SUITE_TMPDIR}/run/atomfs + mkdir -p $ATOMFS_TEST_RUN_DIR + export MY_MNTNSNAME=$(readlink /proc/self/ns/mnt | cut -c 6-15) +} + +function setup() { + export MP=${BATS_TEST_TMPDIR}/testmountpoint + mkdir -p $MP +} + +@test "guestmount works ignoring verity" { + + lxc-usernsexec -s <