Centralize metadata, move mount logic to atomfs pkg, add tests (#23)

* write molecule config to metadata path The molecule config contains the OCI path and tag, which is important if we want to track what container image is mounted at a particular path, and thus what container might need to stop if a verity error is detected in one of the atoms in the molecule it's using. This commit writes that config to a JSON file in the metadata path. Signed-off-by: Michael McCracken <[email protected]> * Move metadir to /run, mount logic into pkg 1 - To make it easy to read the config.json for a given mount, move the metadata path to /run/atomfs/meta/$current-mountns/$munged-mountpt-name/ using the current mount NS name in the path means we can track mounting different images on to the same mountpoint in different mount namespaces. 2 - move mount logic including the metadata dir logic from cmd/atomfs/mount.go to atomfs/molecule.go so that users of the package will also get the same metadir / config.json etc behavior that the `atomfs` tool does. As part of this move, we no longer mount an RO overlay in one place and then either remount another overlay or bind mount to the final target mountpoint. Instead we build one overlay mount for the mountpoint and either it has an upperdir/workdir or not. Signed-off-by: Michael McCracken <[email protected]> * add flag to allow missing verity, enforce it pass through to molecule config, and check to be sure we don't guestmount and ignore verity data without explicitly saying we want to Signed-off-by: Michael McCracken <[email protected]> * verify: return error when no squash mounts found In the guestmount case, we use squashfuse and there is no verity mount source to check. Treat this as a verify error. Signed-off-by: Michael McCracken <[email protected]> * test: add bats tests for mount Add a bats tests suite for mounting and for failing to mount when the images are bad. Uses the ATOMFS_TEST_RUN_DIR env var to avoid polluting your host's /run/atomfs/meta dir. copies the guestmount test from the github yaml into bats and expands it a bit. I apologize for the bash quoting situation in the heredoc in the guestmount tests, forgive me Missing cases: - testing `atomfs verify` on bad images: requires manufacturing a verity image that will mount OK but has a bad block that won't get read until later. I have tested verify with mounted bad images that I mounted with a purposely broken atomfs, but there should be a better way for CI. Signed-off-by: Michael McCracken <[email protected]> * move github test to use bats the existing test is now covered there, and we build our own test image so we can avoid the zothub dep and skopeo dep Signed-off-by: Michael McCracken <[email protected]> * ensure workdir and upperdir are on same fs This redefines the --persist argument to point to a directory where atomfs will create or use both the workdir and the upperdir. So if you run `atomfs mount --persist=foo/` then the persistent writes will end up at foo/persist/. Signed-off-by: Michael McCracken <[email protected]> * add metadir flag to substitute for /run/atomfs In some cases, e.g. when guestmounting in a new userns and mountns, but not chrooted, /run/atomfs may not be writable. In that situation, use the new --metadir flag to all commands to specify a replacement for the default /run/atomfs. This overlaps slightly with the ATOMFS_TEST_RUN_DIR environment variable, which would have the same effect, but should only be used for tests, as it is not discoverable. Signed-off-by: Michael McCracken <[email protected]> --------- Signed-off-by: Michael McCracken <[email protected]>
project-machine · Nov 1, 2024 · eaa7b43 · eaa7b43
1 parent 841e0c2
commit eaa7b43
Show file tree

Hide file tree

Showing 20 changed files with 690 additions and 208 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -51,12 +51,6 @@ jobs:
 
           echo "writing /etc/lxc/lxc-usernet"
           echo "$u veth lxcbr0 100" | sudo tee -a /etc/lxc/lxc-usernet
-      - name: install skopeo
-        run: |
-          mkdir ~/bin
-          wget -O ~/bin/skopeo --progress=dot:mega https://github.com/project-machine/tools/releases/download/v0.0.1/skopeo
-          chmod 755 ~/bin/skopeo
-          sudo cp -f ~/bin/skopeo /usr/bin/skopeo
       - name: lint
         run: |
           make gofmt
@@ -71,15 +65,7 @@ jobs:
           cp ./bin/atomfs atomfs-${{ matrix.os }}
       - name: test
         run: |
-          export PATH=~/bin:$PATH
-          skopeo copy docker://zothub.io/machine/bootkit/bootkit:v0.0.16.230901-squashfs oci:oci:bootkit-squashfs
-          lxc-usernsexec -s << EOF
-          atomfs mount --persist=upper oci:bootkit-squashfs dest
-          [ -d dest/bootkit ]
-          touch dest/zz
-          atomfs umount dest
-          [ -f upper/zz ]
-          EOF
+          make batstest
       - name: Upload code coverage
         uses: codecov/codecov-action@v4
         with:

diff --git a/.gitignore b/.gitignore
@@ -65,3 +65,6 @@ VERSION
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 coverage.txt
+
+# bats test stuff
+/bats-core
diff --git a/Makefile b/Makefile
@@ -9,6 +9,14 @@ ROOT := $(shell git rev-parse --show-toplevel)
 GO_SRC_DIRS := $(shell find . -name "*.go" | xargs -n1 dirname | sort -u)
 GO_SRC := $(shell find . -name "*.go")
 VERSION_LDFLAGS=-X main.Version=$(MAIN_VERSION)
+BATS = $(TOOLS_D)/bin/bats
+BATS_VERSION := v1.10.0
+STACKER = $(TOOLS_D)/bin/stacker
+STACKER_VERSION := v1.0.0
+TOOLS_D := $(ROOT)/tools
+
+export PATH := $(TOOLS_D)/bin:$(PATH)
+
 
 .PHONY: gofmt
 gofmt: .made-gofmt
@@ -23,6 +31,36 @@ atomfs: .made-gofmt $(GO_SRC)
 gotest: $(GO_SRC)
 	go test -coverprofile=coverage.txt -ldflags "$(VERSION_LDFLAGS)"  ./...
 
-clean:
-	rm -f $(ROOT)/bin
-	rm .made-*
+$(STACKER):
+	mkdir -p $(TOOLS_D)/bin
+	wget --progress=dot:giga https://github.com/project-stacker/stacker/releases/download/$(STACKER_VERSION)/stacker
+	chmod +x stacker
+	cp stacker $(TOOLS_D)/bin/
+
+$(BATS):
+	mkdir -p $(TOOLS_D)/bin
+	git clone -b $(BATS_VERSION) https://github.com/bats-core/bats-core.git
+	cd bats-core; ./install.sh $(TOOLS_D)
+	mkdir -p $(ROOT)/test/test_helper
+	git clone --depth 1 https://github.com/bats-core/bats-support $(ROOT)/test/test_helper/bats-support
+	git clone --depth 1 https://github.com/bats-core/bats-assert $(ROOT)/test/test_helper/bats-assert
+	git clone --depth 1 https://github.com/bats-core/bats-file $(ROOT)/test/test_helper/bats-file
+
+batstest: $(BATS) $(STACKER) atomfs test/random.txt
+	cd $(ROOT)/test; sudo $(BATS) --tap --timing priv-*.bats
+	cd $(ROOT)/test; $(BATS) --tap --timing unpriv-*.bats
+
+test/random.txt:
+	dd if=/dev/random of=/dev/stdout count=2048 | base64 > test/random.txt
+
+.PHONY: test toolsclean
+test: gotest batstest
+
+toolsclean:
+	rm -rf $(TOOLS_D)
+	rm -rf $(ROOT)/test/test_helper
+	rm -rf $(ROOT)/bats-core
+
+clean:  toolsclean
+	rm -rf $(ROOT)/bin
+	rm -f .made-*
diff --git a/README.md b/README.md
@@ -42,18 +42,19 @@ ab
 
 ## Implementation details
 
-We create $mountpoint/meta and pass that to `atomfs` as the
-Metadatapath.  We do the readonly `atomfs` molecule mount
-onto $metadir/ro.  Then if a readonly mount is requested
-$metadir/ro is bind mounted onto $metadir.  Otherwise, we create
-$metadir/work and $metadir/upper, and use these to do a rw
-overlay mount of $metadir/ro onto $mountpoint.
+The `atomfs` binary uses the `atomfs` package's Molecule API to mount oci
+images.
+
+Each squashfs layer is mounted separately at a subdir under
+`/run/atomfs/meta/$mountnsid/$mountpoint/`, and then an overlay mount is
+constructed for the specified mountpath. If specified in the config, a writeable
+upperdir is added to the overlay mount.
 
 Note that if you simply call `umount` on the mountpoint, then
 you will be left with all the individual squashfs mounts under
-`dest/mounts/*/`.
+`/run/atomfs/meta/$mountnsid/$mountpoint/`. Use `atomfs umount` instead.
 
 Note that you do need to be root in your namespace in order to
-do the final bind or overlay mount.  (We could get around this
+do the final overlay mount.  (We could get around this
 by using fuse-overlay, but creating a namespace seems overall
 tidy).
diff --git a/cmd/atomfs/mount.go b/cmd/atomfs/mount.go
@@ -1,16 +1,15 @@
 package main
 
 import (
-	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
-	"syscall"
 
+	"github.com/pkg/errors"
 	"github.com/urfave/cli"
-	"golang.org/x/sys/unix"
+
 	"machinerun.io/atomfs"
 	"machinerun.io/atomfs/squashfs"
 )
@@ -22,13 +21,21 @@ var mountCmd = cli.Command{
 	Action:    doMount,
 	Flags: []cli.Flag{
 		cli.StringFlag{
-			Name:  "persist, upper, upperdir",
-			Usage: "Specify a directory to use as writeable overlay (implies --writeable)",
+			Name:  "persist",
+			Usage: "Specify a directory to use for the workdir and upperdir of a writeable overlay (implies --writeable)",
 		},
 		cli.BoolFlag{
 			Name:  "writeable, writable",
 			Usage: "Make the mount writeable using an overlay (ephemeral by default)",
 		},
+		cli.BoolFlag{
+			Name:  "allow-missing-verity",
+			Usage: "Mount even if the image has no verity data",
+		},
+		cli.StringFlag{
+			Name:  "metadir",
+			Usage: "Directory to use for metadata. Use this if /run/atomfs is not writable for some reason.",
+		},
 	},
 }
 
@@ -44,7 +51,7 @@ func findImage(ctx *cli.Context) (string, string, error) {
 	}
 	ocidir := r[0]
 	tag := r[1]
-	if !PathExists(ocidir) {
+	if !atomfs.PathExists(ocidir) {
 		return "", "", fmt.Errorf("oci directory %s does not exist: %w", ocidir, mountUsage(ctx.App.Name))
 	}
 	return ocidir, tag, nil
@@ -70,92 +77,44 @@ func doMount(ctx *cli.Context) error {
 		os.Exit(1)
 	}
 	target := ctx.Args()[1]
-	metadir := filepath.Join(target, "meta")
-
-	complete := false
-
-	defer func() {
-		if !complete {
-			cleanupDest(metadir)
-		}
-	}()
-
-	if PathExists(metadir) {
-		return fmt.Errorf("%q exists: cowardly refusing to mess with it", metadir)
-	}
-
-	if err := EnsureDir(metadir); err != nil {
-		return err
-	}
-
-	rodest := filepath.Join(metadir, "ro")
-	if err = EnsureDir(rodest); err != nil {
-		return err
-	}
-
-	opts := atomfs.MountOCIOpts{
-		OCIDir:       ocidir,
-		MetadataPath: metadir,
-		Tag:          tag,
-		Target:       rodest,
-	}
-
-	mol, err := atomfs.BuildMoleculeFromOCI(opts)
+	absTarget, err := filepath.Abs(target)
 	if err != nil {
 		return err
 	}
 
-	err = mol.Mount(rodest)
+	absOCIDir, err := filepath.Abs(ocidir)
 	if err != nil {
 		return err
 	}
 
-	if ctx.Bool("writeable") || ctx.IsSet("persist") {
-		err = overlay(target, rodest, metadir, ctx)
-	} else {
-		err = bind(target, rodest)
-	}
-
-	complete = err == nil
-	return err
-}
-
-func cleanupDest(metadir string) {
-	fmt.Printf("Failure detected: cleaning up %q", metadir)
-	rodest := filepath.Join(metadir, "ro")
-	if PathExists(rodest) {
-		if err := unix.Unmount(rodest, 0); err != nil {
-			fmt.Printf("Failed unmounting %q: %v", rodest, err)
+	persistPath := ""
+	if ctx.IsSet("persist") {
+		persistPath = ctx.String("persist")
+		if persistPath == "" {
+			return fmt.Errorf("--persist requires an argument")
 		}
 	}
+	opts := atomfs.MountOCIOpts{
+		OCIDir:                 absOCIDir,
+		Tag:                    tag,
+		Target:                 absTarget,
+		AddWriteableOverlay:    ctx.Bool("writeable") || ctx.IsSet("persist"),
+		WriteableOverlayPath:   persistPath,
+		AllowMissingVerityData: ctx.Bool("allow-missing-verity"),
+		MetadataDir:            ctx.String("metadir"), // nil here means /run/atomfs
+	}
 
-	mountsdir := filepath.Join(metadir, "mounts")
-	entries, err := os.ReadDir(mountsdir)
+	mol, err := atomfs.BuildMoleculeFromOCI(opts)
 	if err != nil {
-		fmt.Printf("Failed reading contents of %q: %v", mountsdir, err)
-		os.RemoveAll(metadir)
-		return
+		return errors.Wrapf(err, "couldn't build molecule with opts %+v", opts)
 	}
 
-	wd, err := os.Getwd()
+	err = mol.Mount(target)
 	if err != nil {
-		fmt.Printf("Failed getting working directory")
-		os.RemoveAll(metadir)
+		return errors.Wrapf(err, "couldn't mount molecule at mntpt %q ", target)
 	}
-	for _, e := range entries {
-		n := filepath.Base(e.Name())
-		if n == "workaround" {
-			continue
-		}
-		if strings.HasSuffix(n, ".log") {
-			continue
-		}
-		p := filepath.Join(wd, mountsdir, e.Name())
-		if err := squashUmount(p); err != nil {
-			fmt.Printf("Failed unmounting %q: %v\n", p, err)
-		}
-	}
-	os.RemoveAll(metadir)
+
+	return nil
 }
 
 func RunCommand(args ...string) error {
@@ -177,23 +136,3 @@ func squashUmount(p string) error {
 	}
 	return RunCommand("fusermount", "-u", p)
 }
-
-func overlay(target, rodest, metadir string, ctx *cli.Context) error {
-	workdir := filepath.Join(metadir, "work")
-	if err := EnsureDir(workdir); err != nil {
-		return err
-	}
-	upperdir := filepath.Join(metadir, "persist")
-	if ctx.IsSet("persist") {
-		upperdir = ctx.String("persist")
-	}
-	if err := EnsureDir(upperdir); err != nil {
-		return err
-	}
-	overlayArgs := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s,index=off,userxattr", rodest, upperdir, workdir)
-	return unix.Mount("overlayfs", target, "overlay", 0, overlayArgs)
-}
-
-func bind(target, source string) error {
-	return syscall.Mount(source, target, "", syscall.MS_BIND, "")
-}
diff --git a/cmd/atomfs/umount.go b/cmd/atomfs/umount.go
@@ -7,6 +7,7 @@ import (
 	"syscall"
 
 	"github.com/urfave/cli"
+	"machinerun.io/atomfs"
 	"machinerun.io/atomfs/mount"
 )
 
@@ -15,6 +16,12 @@ var umountCmd = cli.Command{
 	Usage:     "unmount atomfs image",
 	ArgsUsage: "mountpoint",
 	Action:    doUmount,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "metadir",
+			Usage: "Directory to use for metadata. Use this if /run/atomfs is not writable for some reason.",
+		},
+	},
 }
 
 func umountUsage(me string) error {
@@ -43,36 +50,37 @@ func doUmount(ctx *cli.Context) error {
 		}
 	}
 
-	// We expect the argument to be the mountpoint - either a readonly
-	// bind mount, or a writeable overlay.
+	// We expect the argument to be the mountpoint of the overlay
 	err = syscall.Unmount(mountpoint, 0)
 	if err != nil {
 		errs = append(errs, fmt.Errorf("Failed unmounting %s: %v", mountpoint, err))
 	}
 
-	// Now that we've unmounted the mountpoint, we expect the following
-	// under there:
-	// $mountpoint/meta/ro - the original readonly overlay mountpoint
-	// $mountpoint/meta/mounts/* - the original squashfs mounts
-	metadir := filepath.Join(mountpoint, "meta")
-	p := filepath.Join(metadir, "ro")
-	err = syscall.Unmount(p, 0)
+	// We expect the following in the metadir
+	//
+	// $metadir/mounts/* - the original squashfs mounts
+	// $metadir/meta/config.json
+
+	// TODO: want to know mountnsname for a target mountpoint... not for our current proc???
+	mountNSName, err := atomfs.GetMountNSName()
 	if err != nil {
-		errs = append(errs, fmt.Errorf("Failed unmounting RO mountpoint %s: %v", p, err))
+		errs = append(errs, fmt.Errorf("Failed to get mount namespace name"))
 	}
+	metadir := filepath.Join(atomfs.RuntimeDir(ctx.String("metadir")), "meta", mountNSName, atomfs.ReplacePathSeparators(mountpoint))
 
 	mountsdir := filepath.Join(metadir, "mounts")
 	mounts, err := os.ReadDir(mountsdir)
 	if err != nil {
 		errs = append(errs, fmt.Errorf("Failed reading list of mounts: %v", err))
-		return fmt.Errorf("Encountered errors: %#v", errs)
+		return fmt.Errorf("Encountered errors: %v", errs)
 	}
 
 	for _, m := range mounts {
-		p = filepath.Join(mountsdir, m.Name())
+		p := filepath.Join(mountsdir, m.Name())
 		if !m.IsDir() || !isMountpoint(p) {
 			continue
 		}
+
 		err = syscall.Unmount(p, 0)
 		if err != nil {
 			errs = append(errs, fmt.Errorf("Failed unmounting squashfs dir %s: %v", p, err))