csi: delete stale subvolumesnapshot

this commit checks if there are any stale
volumesnapshots, if yes, it deletes the
snapshots

Signed-off-by: yati1998 <[email protected]>

go.mod

@@ -2,9 +2,12 @@ module github.com/rook/kubectl-rook-ceph
 
 go 1.22.0
 
+toolchain go1.22.3
+
 require (
 	github.com/fatih/color v1.17.0
 	github.com/golang/mock v1.6.0
+	github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0
 	github.com/pkg/errors v0.9.1
 	github.com/rook/rook v1.14.8
 	github.com/rook/rook/pkg/apis v0.0.0-20231204200402-5287527732f7

go.sum

@@ -521,6 +521,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1 h1:dQEHhTfi+bSIOSViQrKY9PqJvZenD6tFz+3lPzux58o=
 github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1/go.mod h1:my+EVjOJLeQ9lUR9uVkxRvNNkhO2saSGIgzV8GZT9HY=
 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
+github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 h1:mjQG0Vakr2h246kEDR85U8y8ZhPgT3bguTCajRa/jaw=
+github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
 github.com/libopenstorage/autopilot-api v0.6.1-0.20210128210103-5fbb67948648/go.mod h1:6JLrPbR3ZJQFbUY/+QJMl/aF00YdIrLf8/GWAplgvJs=
 github.com/libopenstorage/openstorage v8.0.0+incompatible/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
 github.com/libopenstorage/operator v0.0.0-20200725001727-48d03e197117/go.mod h1:Qh+VXOB6hj60VmlgsmY+R1w+dFuHK246UueM4SAqZG0=

pkg/filesystem/subvolume.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 	"syscall"
 
+	snapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1"
 	"github.com/rook/kubectl-rook-ceph/pkg/exec"
 	"github.com/rook/kubectl-rook-ceph/pkg/k8sutil"
 	"github.com/rook/kubectl-rook-ceph/pkg/logging"
@@ -42,6 +43,11 @@ type subVolumeInfo struct {
 	state string
 }
 
+type snapshotInfo struct {
+	volumeHandle   string
+	snapshotHandle string
+}
+
 type monitor struct {
 	ClusterID string
 	Monitors  []string
@@ -51,12 +57,14 @@ const (
 	inUse             = "in-use"
 	stale             = "stale"
 	staleWithSnapshot = "stale-with-snapshot"
+	snapshotRetained  = "snapshot-retained"
 )
 
 func List(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool) {
 
 	subvolumeNames := getK8sRefSubvolume(ctx, clientsets)
-	listCephFSSubvolumes(ctx, clientsets, operatorNamespace, clusterNamespace, includeStaleOnly, subvolumeNames)
+	snapshotHandles := getK8sRefSnapshotHandle(ctx, clientsets)
+	listCephFSSubvolumes(ctx, clientsets, operatorNamespace, clusterNamespace, includeStaleOnly, subvolumeNames, snapshotHandles)
 }
 
 // checkForExternalStorage checks if the external mode is enabled.
@@ -152,6 +160,48 @@ func getSubvolumeNameFromPath(path string) (string, error) {
 	return name, nil
 }
 
+// getk8sRefSnapshotHandle returns the snapshothandle for k8s ref of the volume snapshots
+func getK8sRefSnapshotHandle(ctx context.Context, clientsets *k8sutil.Clientsets) map[string]snapshotInfo {
+
+	snapConfig, err := snapclient.NewForConfig(clientsets.KubeConfig)
+	if err != nil {
+		logging.Fatal(err)
+	}
+	snapList, err := snapConfig.VolumeSnapshotContents().List(ctx, v1.ListOptions{})
+	if err != nil {
+		logging.Fatal(fmt.Errorf("Error fetching volumesnapshotcontents: %v\n", err))
+	}
+
+	snapshotHandles := make(map[string]snapshotInfo)
+	for _, snap := range snapList.Items {
+		driverName := snap.Spec.Driver
+		if snap.Status != nil && snap.Status.SnapshotHandle != nil && strings.Contains(driverName, "cephfs.csi.ceph.com") {
+
+			snapshotHandleId := getSnapshotHandleId(*snap.Status.SnapshotHandle)
+			// map the snapshotHandle id to later lookup for the subvol id and
+			// match the subvolume snapshot.
+			snapshotHandles[snapshotHandleId] = snapshotInfo{}
+		}
+	}
+
+	return snapshotHandles
+}
+
+// getSnapshotHandleId gets the id from snapshothandle
+// SnapshotHandle: 0001-0009-rook-ceph-0000000000000001-17b95621-
+// 58e8-4676-bc6a-39e928f19d23
+// SnapshotHandleId: 17b95621-58e8-4676-bc6a-39e928f19d23
+func getSnapshotHandleId(snapshotHandle string) string {
+	// get the snaps id from snapshot handle
+	splitSnapshotHandle := strings.SplitAfterN(snapshotHandle, "-", 6)
+	if len(splitSnapshotHandle) < 6 {
+		return ""
+	}
+	snapshotHandleId := splitSnapshotHandle[len(splitSnapshotHandle)-1]
+
+	return snapshotHandleId
+}
+
 // runCommand checks for the presence of externalcluster and runs the command accordingly.
 func runCommand(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, cmd string, args []string) (string, error) {
 	if checkForExternalStorage(ctx, clientsets, clusterNamespace) {
@@ -164,7 +214,7 @@ func runCommand(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNam
 }
 
 // listCephFSSubvolumes list all the subvolumes
-func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool, subvolumeNames map[string]subVolumeInfo) {
+func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool, subvolumeNames map[string]subVolumeInfo, snapshotHandles map[string]snapshotInfo) {
 
 	// getFilesystem gets the filesystem
 	fsstruct, err := getFileSystem(ctx, clientsets, operatorNamespace, clusterNamespace)
@@ -226,11 +276,12 @@ func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, o
 				} else {
 					// check the state of the stale subvolume
 					// if it is snapshot-retained then skip listing it.
-					if state == "snapshot-retained" {
+					if state == snapshotRetained {
+						status = snapshotRetained
 						continue
 					}
 					// check if the stale subvolume has snapshots.
-					if checkSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs.Name, sv.Name, svg.Name) {
+					if checkSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs.Name, sv.Name, svg.Name, snapshotHandles) {
 						status = staleWithSnapshot
 					}
 
@@ -283,7 +334,8 @@ func getFileSystem(ctx context.Context, clientsets *k8sutil.Clientsets, operator
 }
 
 // checkSnapshot checks if there are any snapshots in the subvolume
-func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, fs, sv, svg string) bool {
+// it also check for the stale snapshot and if found, deletes the snapshot.
+func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, fs, sv, svg string, snapshotHandles map[string]snapshotInfo) bool {
 
 	cmd := "ceph"
 	args := []string{"fs", "subvolume", "snapshot", "ls", fs, sv, svg, "--format", "json"}
@@ -294,6 +346,19 @@ func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operator
 		return false
 	}
 	snap := unMarshaljson(snapList)
+	// check for stale subvolume snapshot
+	// we have the list of snapshothandleid's from the
+	// volumesnapshotcontent. Looking up for snapid in it
+	// will confirm if we have stale snapshot or not.
+	for _, s := range snap {
+		_, snapId := getSnapOmapVal(s.Name)
+		// lookup for the snapid in the k8s snapshot handle list
+		_, ok := snapshotHandles[snapId]
+		if !ok {
+			// delete stale snapshot
+			deleteSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs, sv, svg, s.Name)
+		}
+	}
 	if len(snap) == 0 {
 		return false
 	}
@@ -327,6 +392,19 @@ func unMarshaljson(list string) []fsStruct {
 	return unmarshal
 }
 
+// deleteSnapshot deletes the subvolume snapshot
+func deleteSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, cephClusterNamespace, fs, subvol, svg, snap string) {
+
+	deleteOmapForSnapshot(ctx, clientsets, operatorNamespace, cephClusterNamespace, snap, fs)
+	cmd := "ceph"
+	args := []string{"fs", "subvolume", "snapshot", "rm", fs, subvol, snap, svg}
+
+	_, err := runCommand(ctx, clientsets, operatorNamespace, cephClusterNamespace, cmd, args)
+	if err != nil {
+		logging.Fatal(err, "failed to delete subvolume snapshot of %s/%s/%s/%s", fs, svg, subvol, snap)
+	}
+}
+
 func Delete(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs, subvol, svg string) {
 	k8sSubvolume := getK8sRefSubvolume(ctx, clientsets)
 	_, check := k8sSubvolume[subvol]
@@ -410,6 +488,40 @@ func deleteOmapForSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets,
 	}
 }
 
+// deleteOmapForSnapshot deletes omap object and key for the given snapshot.
+func deleteOmapForSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, snap, fs string) {
+	logging.Info("Deleting the omap object and key for snapshot %q", snap)
+	snapomapkey := getSnapOmapKey(ctx, clientsets, OperatorNamespace, CephClusterNamespace, snap, fs)
+	snapomapval, _ := getSnapOmapVal(snap)
+	poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs)
+	if err != nil || poolName == "" {
+		logging.Fatal(fmt.Errorf("pool name not found: %q", err))
+	}
+	cmd := "rados"
+	if snapomapval != "" {
+		args := []string{"rm", snapomapval, "-p", poolName, "--namespace", "csi"}
+
+		// remove omap object.
+		_, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args)
+		if err != nil {
+			logging.Fatal(err, "failed to remove omap object for snapshot %q", snap)
+		}
+		logging.Info("omap object:%q deleted", snapomapval)
+
+	}
+	if snapomapkey != "" {
+		args := []string{"rmomapkey", "csi.snaps.default", snapomapkey, "-p", poolName, "--namespace", "csi"}
+
+		// remove omap key.
+		_, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args)
+		if err != nil {
+			logging.Fatal(err, "failed to remove omap key for snapshot %q", snap)
+		}
+		logging.Info("omap key:%q deleted", snapomapkey)
+
+	}
+}
+
 // getOmapKey gets the omap key and value details for a given subvolume.
 // Deletion of omap object required the subvolumeName which is of format
 // csi.volume.subvolume, where subvolume is the name of subvolume that needs to be
@@ -438,6 +550,36 @@ func getOmapKey(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNam
 	return omapkey
 }
 
+// getSnapOmapKey gets the omap key and value details for a given snapshot.
+// Deletion of omap object required the snapshotName which is of format
+// csi.snap.snapid.
+// similarly to delete of omap key requires csi.snap.ompakey, where
+// omapkey is the snapshotcontent name which is extracted the omap object.
+func getSnapOmapKey(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, cephClusterNamespace, snap, fs string) string {
+
+	poolName, err := getMetadataPoolName(ctx, clientsets, operatorNamespace, cephClusterNamespace, fs)
+	if err != nil || poolName == "" {
+		logging.Fatal(fmt.Errorf("pool name not found: %q", err))
+	}
+	snapomapval, _ := getSnapOmapVal(snap)
+
+	args := []string{"getomapval", snapomapval, "csi.snapname", "-p", poolName, "--namespace", "csi", "/dev/stdout"}
+	cmd := "rados"
+	snapshotcontentname, err := runCommand(ctx, clientsets, operatorNamespace, cephClusterNamespace, cmd, args)
+	if snapshotcontentname == "" && err == nil {
+		logging.Info("No snapshot content found for snapshot")
+		return ""
+	}
+	if err != nil {
+		logging.Fatal(fmt.Errorf("Error getting snapshot content for snapshot %s: %s", snap, err))
+	}
+	// omap key is for format csi.snap.snapshotcontent-fca205e5-8788-4132-979c-e210c0133182
+	// hence, attaching pvname to required prefix.
+	snapomapkey := "csi.snap." + snapshotcontentname
+
+	return snapomapkey
+}
+
 // getNfsClusterName returns the cluster name from the omap.
 // csi.nfs.cluster
 // value (26 bytes) :
@@ -511,3 +653,19 @@ func getOmapVal(subVol string) (string, string) {
 
 	return omapval, subvolId
 }
+
+// func getSnapOmapVal is used to get the omapval from the given snapshot
+// omapval is of format csi.snap.427774b4-340b-11ed-8d66-0242ac110005
+// which is similar to volume name csi-snap-427774b4-340b-11ed-8d66-0242ac110005
+// hence, replacing 'csi-snap-' to 'csi.snap.'
+func getSnapOmapVal(snap string) (string, string) {
+
+	splitSnap := strings.SplitAfterN(snap, "-", 3)
+	if len(splitSnap) < 3 {
+		return "", ""
+	}
+	snapId := splitSnap[len(splitSnap)-1]
+	snapomapval := "csi.snap." + snapId
+
+	return snapomapval, snapId
+}

pkg/filesystem/subvolume_test.go

@@ -101,3 +101,72 @@ func TestGetSubvolumeNameFromPath(t *testing.T) {
 		})
 	}
 }
+
+func TestGetSnapOmapVal(t *testing.T) {
+
+	tests := []struct {
+		name   string
+		val    string
+		snapid string
+	}{
+		{
+			name:   "csi-snap-427774b4-340b-11ed-8d66-0242ac110005",
+			val:    "csi.snap.427774b4-340b-11ed-8d66-0242ac110005",
+			snapid: "427774b4-340b-11ed-8d66-0242ac110005",
+		},
+		{
+			name:   "",
+			val:    "",
+			snapid: "",
+		},
+		{
+			name:   "csi-427774b4-340b-11ed-8d66-0242ac11000",
+			val:    "csi.snap.340b-11ed-8d66-0242ac11000",
+			snapid: "340b-11ed-8d66-0242ac11000",
+		},
+		{
+			name:   "csi-427774b440b11ed8d660242ac11000",
+			val:    "",
+			snapid: "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if val, snapid := getSnapOmapVal(tt.name); val != tt.val && snapid != tt.snapid {
+				t.Errorf("getSnapOmapVal()= got val %v, want val %v,got snapid %v want snapid %v", val, tt.val, snapid, tt.snapid)
+			}
+		})
+	}
+}
+
+func TestGetSnapshotHandleId(t *testing.T) {
+
+	tests := []struct {
+		name string
+		val  string
+	}{
+		{
+			name: "0001-0009-rook-ceph-0000000000000001-17b95621-58e8-4676-bc6a-39e928f19d23",
+			val:  "17b95621-58e8-4676-bc6a-39e928f19d23",
+		},
+		{
+			name: "",
+			val:  "",
+		},
+		{
+			name: "0001-0009-rook-0000000000000001-17b95621-58e8-4676-bc6a-39e928f19d23",
+			val:  "58e8-4676-bc6a-39e928f19d23",
+		},
+		{
+			name: "rook-427774b440b11ed8d660242ac11000",
+			val:  "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if val := getSnapshotHandleId(tt.name); val != tt.val {
+				assert.Equal(t, val, tt.val)
+			}
+		})
+	}
+}

tests/github-action-helper.sh

@@ -241,7 +241,7 @@ install_minikube_with_none_driver() {
 }
 
 install_external_snapshotter() {
-  EXTERNAL_SNAPSHOTTER_VERSION=7.0.2
+  EXTERNAL_SNAPSHOTTER_VERSION=8.0.1
   curl -L "https://github.com/kubernetes-csi/external-snapshotter/archive/refs/tags/v${EXTERNAL_SNAPSHOTTER_VERSION}.zip" -o external-snapshotter.zip
   unzip -d /tmp external-snapshotter.zip
   cd "/tmp/external-snapshotter-${EXTERNAL_SNAPSHOTTER_VERSION}"