Auto import images for containerd image store

.github/workflows/e2e.yaml

@@ -40,7 +40,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        etest: [startup, s3, btrfs, externalip, privateregistry, embeddedmirror, wasm]
+        etest: [autoimport, startup, s3, btrfs, externalip, privateregistry, embeddedmirror, wasm]
       max-parallel: 3
     steps:
       - name: "Checkout"

docs/adrs/add-auto-import-containerd.md

@@ -0,0 +1,54 @@
+# Easy way for auto adding images to k3s
+
+Date: 2024-10-2
+
+## Status
+
+Proposed
+
+## Context
+
+Since the feature for embedded registry, the users appeared with a question about having to manually import images, specially in edge environments.
+
+As a result, there is a need for a folder who can handle this action, where every image there will be watched by a controller for changes or new images, this new images or new changes will be added to the containerd image store.
+
+The controller will watch the agent/images folder that is the default folder for the images, as the first iteration about the controller he will mainly work with the default image folder, but in the future we can set to watch more folders.
+
+The main idea for the controller is to create a map for the file infos maintaining the state for the files, with that we can see if a file was modified and if the size changed.
+
+### Map to handle the state from the files
+
+This map will have the entire filepath of the file in the `key` value, since we can get the value from the key with only the `event.Name`
+
+```go
+    map[string]fs.FileInfo
+```
+
+### Why use fsnotify
+
+With this library we can easily use for any linux distros without the need to port for a specify distro and can also run in windows.
+
+The main idea for the watch will be taking care of the last time that was modified the image file.
+
+fsnotify has a great toolset for handling changes in files, since the code will have a channel to receive events such as CREATE, RENAME, REMOVE and WRITE.
+
+### How the controller will work with the events
+
+When the controller receive a event saying that a file was created, he will add to the map and import the images if the event that he has received is not a directory and then import the image.
+
+When the controller receive a event saying that a file was writen, he will verify if the file has the size changed and if the file has the time modified based on the time and size from the state.
+
+When the controller receive a event saying that a file was renamed, or removed, he will delete this file from the state. when a file is renamed, it is created a new file with the same infos but with a the new name, so the watcher will sent for the controller a event saying that a file was created.
+
+## Decision
+
+- Decided
+
+## Consequences
+
+Good:
+- Better use of embedded containerd image store.
+- Fsnotify it's a indirect dependency that upstream uses
+
+Bad:
+- The need for another dependency

go.mod

@@ -90,6 +90,7 @@ require (
 	github.com/docker/docker v27.1.1+incompatible
 	github.com/erikdubbelboer/gspt v0.0.0-20190125194910-e68493906b83
 	github.com/flannel-io/flannel v0.25.7
+	github.com/fsnotify/fsnotify v1.7.0
 	github.com/go-bindata/go-bindata v3.1.2+incompatible
 	github.com/go-logr/logr v1.4.2
 	github.com/go-logr/stdr v1.2.3-0.20220714215716-96bad1d688c5
@@ -165,6 +166,7 @@ require (
 	k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3
 	sigs.k8s.io/cri-tools v0.0.0-00010101000000-000000000000
 	sigs.k8s.io/yaml v1.4.0
+	github.com/google/go-containerregistry v0.20.2
 )
 
 require (
@@ -252,7 +254,6 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/flynn/noise v1.1.0 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
-	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.2 // indirect
@@ -272,7 +273,6 @@ require (
 	github.com/google/cel-go v0.20.1 // indirect
 	github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
-	github.com/google/go-containerregistry v0.20.2 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
 	github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect

pkg/agent/containerd/containerd.go

@@ -18,6 +18,7 @@ import (
 	"github.com/containerd/containerd/pkg/cri/constants"
 	"github.com/containerd/containerd/pkg/cri/labels"
 	"github.com/containerd/containerd/reference/docker"
+	reference "github.com/google/go-containerregistry/pkg/name"
 	"github.com/k3s-io/k3s/pkg/agent/cri"
 	util2 "github.com/k3s-io/k3s/pkg/agent/util"
 	"github.com/k3s-io/k3s/pkg/daemons/config"
@@ -115,24 +116,6 @@ func Run(ctx context.Context, cfg *config.Node) error {
 // any .txt files are processed as a list of images that should be pre-pulled from remote registries.
 // If configured, imported images are retagged as being pulled from additional registries.
 func PreloadImages(ctx context.Context, cfg *config.Node) error {
-	fileInfo, err := os.Stat(cfg.Images)
-	if os.IsNotExist(err) {
-		return nil
-	} else if err != nil {
-		logrus.Errorf("Unable to find images in %s: %v", cfg.Images, err)
-		return nil
-	}
-
-	if !fileInfo.IsDir() {
-		return nil
-	}
-
-	fileInfos, err := os.ReadDir(cfg.Images)
-	if err != nil {
-		logrus.Errorf("Unable to read images in %s: %v", cfg.Images, err)
-		return nil
-	}
-
 	client, err := Client(cfg.Containerd.Address)
 	if err != nil {
 		return err
@@ -162,6 +145,28 @@ func PreloadImages(ctx context.Context, cfg *config.Node) error {
 		return errors.Wrap(err, "failed to clear pinned labels")
 	}
 
+	go watchImages(ctx, cfg)
+
+	// After setting the watcher, connections and everything, k3s will see if the images folder is already created
+	// if the folder its already created, it will load the images
+	fileInfo, err := os.Stat(cfg.Images)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		logrus.Errorf("Unable to find images in %s: %v", cfg.Images, err)
+		return nil
+	}
+
+	if !fileInfo.IsDir() {
+		return nil
+	}
+
+	fileInfos, err := os.ReadDir(cfg.Images)
+	if err != nil {
+		logrus.Errorf("Unable to read images in %s: %v", cfg.Images, err)
+		return nil
+	}
+
 	for _, fileInfo := range fileInfos {
 		if fileInfo.IsDir() {
 			continue
@@ -176,6 +181,7 @@ func PreloadImages(ctx context.Context, cfg *config.Node) error {
 		}
 		logrus.Infof("Imported images from %s in %s", filePath, time.Since(start))
 	}
+
 	return nil
 }
 
@@ -214,7 +220,7 @@ func preloadFile(ctx context.Context, cfg *config.Node, client *containerd.Clien
 		}
 	}
 
-	if err := labelImages(ctx, client, images); err != nil {
+	if err := labelImages(ctx, client, images, filepath.Base(filePath)); err != nil {
 		return errors.Wrap(err, "failed to add pinned label to images")
 	}
 	if err := retagImages(ctx, client, images, cfg.AgentConfig.AirgapExtraRegistry); err != nil {
@@ -265,7 +271,7 @@ func clearLabels(ctx context.Context, client *containerd.Client) error {
 
 // labelImages adds labels to the listed images, indicating that they
 // are pinned by k3s and should not be pruned.
-func labelImages(ctx context.Context, client *containerd.Client, images []images.Image) error {
+func labelImages(ctx context.Context, client *containerd.Client, images []images.Image, fileName string) error {
 	var errs []error
 	imageService := client.ImageService()
 	for i, image := range images {
@@ -277,6 +283,7 @@ func labelImages(ctx context.Context, client *containerd.Client, images []images
 		if image.Labels == nil {
 			image.Labels = map[string]string{}
 		}
+
 		image.Labels[k3sPinnedImageLabelKey] = k3sPinnedImageLabelValue
 		image.Labels[labels.PinnedImageLabelKey] = labels.PinnedImageLabelValue
 		updatedImage, err := imageService.Update(ctx, image, "labels")
@@ -354,6 +361,16 @@ func prePullImages(ctx context.Context, client *containerd.Client, imageClient r
 	for scanner.Scan() {
 		name := strings.TrimSpace(scanner.Text())
 
+		if name == "" {
+			continue
+		}
+
+		// the options in the reference.ParseReference are for filtering only strings that cannot be seen as a possible image
+		if _, err := reference.ParseReference(name, reference.WeakValidation, reference.Insecure); err != nil {
+			logrus.Errorf("Failed to parse image reference %q: %v", name, err)
+			continue
+		}
+
 		if status, err := imageClient.ImageStatus(ctx, &runtimeapi.ImageStatusRequest{
 			Image: &runtimeapi.ImageSpec{
 				Image: name,