New container locator for docker/k8s on linux

go.mod

@@ -88,6 +88,7 @@ require (
 	k8s.io/apimachinery v0.29.4
 	k8s.io/client-go v0.29.4
 	k8s.io/kube-aggregator v0.29.4
+	k8s.io/mount-utils v0.29.2
 	sigs.k8s.io/controller-runtime v0.17.3
 )
 
@@ -257,6 +258,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/sys/mountinfo v0.6.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mozillazg/docker-credential-acr-helper v0.3.0 // indirect

go.sum

@@ -1222,6 +1222,8 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
 github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78=
+github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
 github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -2237,6 +2239,8 @@ k8s.io/kube-aggregator v0.29.4 h1:yT7vYtwIag4G8HNrktYZ3qz6p6oHKronMAXOw4eQ2WQ=
 k8s.io/kube-aggregator v0.29.4/go.mod h1:zBfe4iXXmw5HinNgN0JoAu5rpXdyCUvRfG99+FVOd68=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
+k8s.io/mount-utils v0.29.2 h1:FrUfgvOo63nqJRPXKoqN/DW1lMnR/y0pzpFErKh6p2o=
+k8s.io/mount-utils v0.29.2/go.mod h1:9IWJTMe8tG0MYMLEp60xK9GYVeCdA3g4LowmnVi+t9Y=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

pkg/agent/plugin/workloadattestor/docker/docker.go

@@ -135,7 +135,7 @@ func (p *Plugin) Configure(_ context.Context, req *configv1.ConfigureRequest) (*
 		return nil, status.Errorf(codes.InvalidArgument, "unknown configurations detected: %s", strings.Join(keys, ","))
 	}
 
-	containerHelper, err := createHelper(config)
+	containerHelper, err := createHelper(config, p.log)
 	if err != nil {
 		return nil, err
 	}

pkg/agent/plugin/workloadattestor/docker/docker_posix.go

@@ -5,11 +5,15 @@ package docker
 import (
 	"errors"
 	"fmt"
+	"io"
+	"os"
+	"path/filepath"
 	"regexp"
 
 	"github.com/hashicorp/go-hclog"
 	"github.com/spiffe/spire/pkg/agent/common/cgroups"
 	"github.com/spiffe/spire/pkg/agent/plugin/workloadattestor/docker/cgroup"
+	"github.com/spiffe/spire/pkg/common/containerinfo"
 )
 
 type OSConfig struct {
@@ -19,36 +23,67 @@ type OSConfig struct {
 	// ContainerIDCGroupMatchers is a list of patterns used to discover container IDs from cgroup entries.
 	// See the documentation for cgroup.NewContainerIDFinder in the cgroup subpackage for more information. (Unix)
 	ContainerIDCGroupMatchers []string `hcl:"container_id_cgroup_matchers" json:"container_id_cgroup_matchers"`
+
+	// UseNewContainerLocator, if true, uses the new container locator
+	// mechanism instead of cgroup matchers. Currently defaults to false if
+	// unset. This will default to true in a future release. (Unix)
+	UseNewContainerLocator *bool `hcl:"use_new_container_locator"`
+
+	// Used by tests to use a fake /proc directory instead of the real one
+	rootDir string
 }
 
-func createHelper(c *dockerPluginConfig) (*containerHelper, error) {
-	var containerIDFinder cgroup.ContainerIDFinder = &defaultContainerIDFinder{}
-	var err error
-	if len(c.ContainerIDCGroupMatchers) > 0 {
+func createHelper(c *dockerPluginConfig, log hclog.Logger) (*containerHelper, error) {
+	var containerIDFinder cgroup.ContainerIDFinder
+
+	switch {
+	case c.UseNewContainerLocator != nil && *c.UseNewContainerLocator:
+		log.Info("Using the new container locator")
+	case len(c.ContainerIDCGroupMatchers) > 0:
+		log.Info("Using the legacy container locator with custom cgroup matchers. The new locator will be enabled by default in a future release. Consider using it now by setting `use_new_container_locator=true`.")
+		var err error
 		containerIDFinder, err = cgroup.NewContainerIDFinder(c.ContainerIDCGroupMatchers)
 		if err != nil {
 			return nil, err
 		}
+	default:
+		log.Info("Using the legacy container locator. The new locator will be enabled by default in a future release. Consider using it now by setting `use_new_container_locator=true`.")
+		containerIDFinder = &defaultContainerIDFinder{}
+	}
+
+	rootDir := c.rootDir
+	if rootDir == "" {
+		rootDir = "/"
 	}
 
 	return &containerHelper{
-		fs:                cgroups.OSFileSystem{},
+		rootDir:           rootDir,
 		containerIDFinder: containerIDFinder,
 	}, nil
 }
 
+type dirFS string
+
+func (d dirFS) Open(p string) (io.ReadCloser, error) {
+	return os.Open(filepath.Join(string(d), p))
+}
+
 type containerHelper struct {
+	rootDir           string
 	containerIDFinder cgroup.ContainerIDFinder
-	fs                cgroups.FileSystem
 }
 
-func (h *containerHelper) getContainerID(pID int32, _ hclog.Logger) (string, error) {
-	cgroupList, err := cgroups.GetCgroups(pID, h.fs)
-	if err != nil {
-		return "", err
+func (h *containerHelper) getContainerID(pID int32, log hclog.Logger) (string, error) {
+	if h.containerIDFinder != nil {
+		cgroupList, err := cgroups.GetCgroups(pID, dirFS(h.rootDir))
+		if err != nil {
+			return "", err
+		}
+		return getContainerIDFromCGroups(h.containerIDFinder, cgroupList)
 	}
 
-	return getContainerIDFromCGroups(h.containerIDFinder, cgroupList)
+	extractor := containerinfo.Extractor{RootDir: h.rootDir}
+	return extractor.GetContainerID(int(pID), log)
 }
 
 func getDockerHost(c *dockerPluginConfig) string {

pkg/agent/plugin/workloadattestor/docker/docker_posix_test.go

@@ -3,19 +3,19 @@
 package docker
 
 import (
-	"io"
 	"os"
-	"strings"
+	"path/filepath"
 	"testing"
 
 	dockerclient "github.com/docker/docker/client"
-	"github.com/spiffe/spire/pkg/agent/common/cgroups"
 	"github.com/spiffe/spire/pkg/agent/plugin/workloadattestor/docker/cgroup"
+	"github.com/spiffe/spire/test/spiretest"
 	"github.com/stretchr/testify/require"
 )
 
 const (
-	testCgroupEntries = "10:devices:/docker/6469646e742065787065637420616e796f6e6520746f20726561642074686973"
+	testCgroupEntries   = "10:devices:/docker/6469646e742065787065637420616e796f6e6520746f20726561642074686973"
+	defaultPluginConfig = "use_new_container_locator = true"
 )
 
 func TestContainerExtraction(t *testing.T) {
@@ -75,8 +75,7 @@ func TestContainerExtraction(t *testing.T) {
 	for _, tt := range tests {
 		tt := tt // alias loop variable as it is used in the closure
 		t.Run(tt.desc, func(t *testing.T) {
-			fs := newFakeFileSystem(tt.cgroups)
-
+			withRootDirOpt := prepareRootDirOpt(t, tt.cgroups)
 			var d Docker = dockerError{}
 			if tt.hasMatch {
 				d = fakeContainer{
@@ -88,7 +87,7 @@ func TestContainerExtraction(t *testing.T) {
 				t,
 				withConfig(t, tt.cfg), // this must be the first option
 				withDocker(d),
-				withFileSystem(fs),
+				withRootDirOpt,
 			)
 
 			selectorValues, err := doAttest(t, p)
@@ -110,12 +109,14 @@ func TestContainerExtraction(t *testing.T) {
 }
 
 func TestCgroupFileNotFound(t *testing.T) {
-	p := newTestPlugin(t, withFileSystem(FakeFileSystem{}))
+	p := newTestPlugin(t, withRootDir(spiretest.TempDir(t)))
 
+	// The new container info extraction code does not consider a missing file
+	// to be an error. It just won't return any container ID so attestation
+	// won't produce any selectors.
 	selectorValues, err := doAttest(t, p)
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "file does not exist")
-	require.Nil(t, selectorValues)
+	require.NoError(t, err)
+	require.Empty(t, selectorValues)
 }
 
 func TestDockerConfigPosix(t *testing.T) {
@@ -148,17 +149,27 @@ container_id_cgroup_matchers = [
 }
 
 func verifyConfigDefault(t *testing.T, c *containerHelper) {
-	require.Equal(t, &defaultContainerIDFinder{}, c.containerIDFinder)
+	// The unit tests configure the plugin to use the new container info
+	// extraction code so the legacy finder should be set to nil.
+	require.Nil(t, c.containerIDFinder)
+}
+
+func withDefaultDataOpt(tb testing.TB) testPluginOpt {
+	return prepareRootDirOpt(tb, testCgroupEntries)
 }
 
-func withDefaultDataOpt() testPluginOpt {
-	fs := newFakeFileSystem(testCgroupEntries)
-	return withFileSystem(fs)
+func prepareRootDirOpt(tb testing.TB, cgroups string) testPluginOpt {
+	rootDir := spiretest.TempDir(tb)
+	procPidPath := filepath.Join(rootDir, "proc", "123")
+	require.NoError(tb, os.MkdirAll(procPidPath, 0755))
+	cgroupsPath := filepath.Join(procPidPath, "cgroup")
+	require.NoError(tb, os.WriteFile(cgroupsPath, []byte(cgroups), 0600))
+	return withRootDir(rootDir)
 }
 
-func withFileSystem(m cgroups.FileSystem) testPluginOpt {
+func withRootDir(dir string) testPluginOpt {
 	return func(p *Plugin) {
-		p.c.fs = m
+		p.c.rootDir = dir
 	}
 }
 
@@ -169,23 +180,3 @@ func withConfig(t *testing.T, cfg string) testPluginOpt {
 		require.NoError(t, err)
 	}
 }
-
-func newFakeFileSystem(cgroups string) FakeFileSystem {
-	return FakeFileSystem{
-		Files: map[string]string{
-			"/proc/123/cgroup": cgroups,
-		},
-	}
-}
-
-type FakeFileSystem struct {
-	Files map[string]string
-}
-
-func (fs FakeFileSystem) Open(path string) (io.ReadCloser, error) {
-	data, ok := fs.Files[path]
-	if !ok {
-		return nil, os.ErrNotExist
-	}
-	return io.NopCloser(strings.NewReader(data)), nil
-}

pkg/agent/plugin/workloadattestor/docker/docker_test.go

@@ -76,7 +76,7 @@ func TestDockerSelectors(t *testing.T) {
 				Env:    tt.mockEnv,
 			}
 
-			p := newTestPlugin(t, withDocker(d), withDefaultDataOpt())
+			p := newTestPlugin(t, withDocker(d), withDefaultDataOpt(t))
 
 			selectorValues, err := doAttest(t, p)
 			require.NoError(t, err)
@@ -89,7 +89,7 @@ func TestDockerSelectors(t *testing.T) {
 func TestDockerError(t *testing.T) {
 	p := newTestPlugin(
 		t,
-		withDefaultDataOpt(),
+		withDefaultDataOpt(t),
 		withDocker(dockerError{}),
 		withDisabledRetryer(),
 	)
@@ -107,7 +107,7 @@ func TestDockerErrorRetries(t *testing.T) {
 		t,
 		withMockClock(mockClock),
 		withDocker(dockerError{}),
-		withDefaultDataOpt(),
+		withDefaultDataOpt(t),
 	)
 
 	go func() {
@@ -131,7 +131,7 @@ func TestDockerErrorContextCancel(t *testing.T) {
 	p := newTestPlugin(
 		t,
 		withMockClock(mockClock),
-		withDefaultDataOpt(),
+		withDefaultDataOpt(t),
 	)
 
 	ctx, cancel := context.WithCancel(context.Background())
@@ -248,7 +248,7 @@ func withDisabledRetryer() testPluginOpt {
 
 func newTestPlugin(t *testing.T, opts ...testPluginOpt) *Plugin {
 	p := New()
-	err := doConfigure(t, p, "")
+	err := doConfigure(t, p, defaultPluginConfig)
 	require.NoError(t, err)
 
 	for _, o := range opts {

pkg/agent/plugin/workloadattestor/docker/docker_windows.go

@@ -14,7 +14,7 @@ type OSConfig struct {
 	DockerHost string `hcl:"docker_host" json:"docker_host"`
 }
 
-func createHelper(*dockerPluginConfig) (*containerHelper, error) {
+func createHelper(*dockerPluginConfig, hclog.Logger) (*containerHelper, error) {
 	return &containerHelper{
 		ph: process.CreateHelper(),
 	}, nil

pkg/agent/plugin/workloadattestor/docker/docker_windows_test.go

@@ -12,6 +12,10 @@ import (
 	"google.golang.org/grpc/codes"
 )
 
+const (
+	defaultPluginConfig = ""
+)
+
 func TestFailToGetContainerID(t *testing.T) {
 	h := &fakeProcessHelper{
 		err: errors.New("oh no"),
@@ -50,7 +54,7 @@ func verifyConfigDefault(t *testing.T, c *containerHelper) {
 	require.NotNil(t, c.ph)
 }
 
-func withDefaultDataOpt() testPluginOpt {
+func withDefaultDataOpt(testing.TB) testPluginOpt {
 	h := &fakeProcessHelper{
 		containerID: testContainerID,
 	}