From bd4100e499f669055b312896c5793f190abcc8d3 Mon Sep 17 00:00:00 2001 From: yati1998 Date: Thu, 4 Jul 2024 14:17:14 +0530 Subject: [PATCH] csi: delete stale subvolumesnapshot this commit checks if there are any stale volumesnapshots, if yes, it deletes the snapshots Signed-off-by: yati1998 --- go.mod | 11 ++- go.sum | 24 +++--- pkg/filesystem/subvolume.go | 153 ++++++++++++++++++++++++++++++++++-- 3 files changed, 169 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 57585b22..79c4df1e 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,22 @@ module github.com/rook/kubectl-rook-ceph -go 1.21 +go 1.22.0 + +toolchain go1.22.3 require ( github.com/fatih/color v1.17.0 github.com/golang/mock v1.6.0 + github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 github.com/pkg/errors v0.9.1 github.com/rook/rook v1.14.7 github.com/rook/rook/pkg/apis v0.0.0-20231204200402-5287527732f7 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.29.3 - k8s.io/apimachinery v0.29.3 - k8s.io/client-go v0.29.3 + k8s.io/api v0.30.0 + k8s.io/apimachinery v0.30.0 + k8s.io/client-go v0.30.0 ) require ( diff --git a/go.sum b/go.sum index 9ec1bbd4..9dc3e90e 100644 --- a/go.sum +++ b/go.sum @@ -520,6 +520,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1 h1:dQEHhTfi+bSIOSViQrKY9PqJvZenD6tFz+3lPzux58o= github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1/go.mod h1:my+EVjOJLeQ9lUR9uVkxRvNNkhO2saSGIgzV8GZT9HY= github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys= +github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0 h1:j3YK74myEQRxR/srciTpOrm221SAvz6J5OVWbyfeXFo= +github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0/go.mod h1:FlyYFe32mPxKEPaRXKNxfX576d1AoCzstYDoOOnyMA4= +github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 h1:mjQG0Vakr2h246kEDR85U8y8ZhPgT3bguTCajRa/jaw= +github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y= github.com/libopenstorage/autopilot-api v0.6.1-0.20210128210103-5fbb67948648/go.mod h1:6JLrPbR3ZJQFbUY/+QJMl/aF00YdIrLf8/GWAplgvJs= github.com/libopenstorage/openstorage v8.0.0+incompatible/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc= github.com/libopenstorage/operator v0.0.0-20200725001727-48d03e197117/go.mod h1:Qh+VXOB6hj60VmlgsmY+R1w+dFuHK246UueM4SAqZG0= @@ -605,8 +609,8 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= -github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= +github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -620,8 +624,8 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= -github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= -github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= +github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= github.com/openshift/api v0.0.0-20210105115604-44119421ec6b/go.mod h1:aqU5Cq+kqKKPbDMqxo9FojgDeSpNJI7iuskjXjtojDg= github.com/openshift/api v0.0.0-20240301093301-ce10821dc999 h1:+S998xHiJApsJZjRAO8wyedU9GfqFd8mtwWly6LqHDo= github.com/openshift/api v0.0.0-20240301093301-ce10821dc999/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= @@ -1398,8 +1402,8 @@ k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= -k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= -k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= +k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA= +k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE= k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= k8s.io/apiextensions-apiserver v0.18.3/go.mod h1:TMsNGs7DYpMXd+8MOCX8KzPOCx8fnZMoIGB24m03+JE= k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= @@ -1412,8 +1416,8 @@ k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRp k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= -k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= +k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= +k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/apiserver v0.18.3/go.mod h1:tHQRmthRPLUtwqsOnJJMoI8SW3lnoReZeE861lH8vUw= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw= @@ -1422,8 +1426,8 @@ k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY= k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= -k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= -k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= +k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= +k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= diff --git a/pkg/filesystem/subvolume.go b/pkg/filesystem/subvolume.go index 5aef3df9..af3dd876 100644 --- a/pkg/filesystem/subvolume.go +++ b/pkg/filesystem/subvolume.go @@ -22,6 +22,7 @@ import ( "fmt" "strings" + snapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1" "github.com/rook/kubectl-rook-ceph/pkg/exec" "github.com/rook/kubectl-rook-ceph/pkg/k8sutil" "github.com/rook/kubectl-rook-ceph/pkg/logging" @@ -40,6 +41,11 @@ type subVolumeInfo struct { state string } +type snapshotInfo struct { + volumehandle string + snapshothandle string +} + type monitor struct { ClusterID string Monitors []string @@ -49,12 +55,14 @@ const ( inUse = "in-use" stale = "stale" staleWithSnapshot = "stale-with-snapshot" + snapshotRetained = "snapshot-retained" ) func List(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool) { subvolumeNames := getK8sRefSubvolume(ctx, clientsets) - listCephFSSubvolumes(ctx, clientsets, operatorNamespace, clusterNamespace, includeStaleOnly, subvolumeNames) + snapshotHandles := getK8sRefSnapshotHandle(ctx, clientsets) + listCephFSSubvolumes(ctx, clientsets, operatorNamespace, clusterNamespace, includeStaleOnly, subvolumeNames, snapshotHandles) } // checkForExternalStorage checks if the external mode is enabled. @@ -131,6 +139,36 @@ func getK8sRefSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets) map return subvolumeNames } +// getk8sRefSnapshotHandle returns the snapshothandle for k8s ref of the volume snapshots +func getK8sRefSnapshotHandle(ctx context.Context, clientsets *k8sutil.Clientsets) map[string]snapshotInfo { + + Snap, err := snapclient.NewForConfig(clientsets.KubeConfig) + if err != nil { + logging.Fatal(err) + } + snapList, err := Snap.VolumeSnapshotContents().List(ctx, v1.ListOptions{}) + if err != nil { + logging.Fatal(fmt.Errorf("Error fetching volumesnapshotcontents: %v\n", err)) + } + + snapshotHandles := make(map[string]snapshotInfo) + for _, snap := range snapList.Items { + if snap.Status != nil && snap.Status.SnapshotHandle != nil { + // get the snaps id from snapshot handle + splitSnapshotHandle := strings.SplitAfterN(*snap.Status.SnapshotHandle, "-", 6) + if len(splitSnapshotHandle) < 6 { + return nil + } + snapshotHandleId := splitSnapshotHandle[len(splitSnapshotHandle)-1] + // map the volumeHandle id to later lookup for the subvol id and + // match the subvolume snapshot. + snapshotHandles[snapshotHandleId] = snapshotInfo{} + } + } + + return snapshotHandles +} + // runCommand checks for the presence of externalcluster and runs the command accordingly. func runCommand(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, cmd string, args []string) (string, error) { if checkForExternalStorage(ctx, clientsets, clusterNamespace) { @@ -143,7 +181,7 @@ func runCommand(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNam } // listCephFSSubvolumes list all the subvolumes -func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool, subvolumeNames map[string]subVolumeInfo) { +func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool, subvolumeNames map[string]subVolumeInfo, snapshotHandles map[string]snapshotInfo) { // getFilesystem gets the filesystem fsstruct, err := getFileSystem(ctx, clientsets, operatorNamespace, clusterNamespace) @@ -195,11 +233,11 @@ func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, o // check the state of the stale subvolume // if it is snapshot-retained then skip listing it. if state == "snapshot-retained" { - status = state + status = snapshotRetained continue } // check if the stale subvolume has snapshots. - if checkSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs.Name, sv.Name, svg.Name) { + if checkSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs.Name, sv.Name, svg.Name, snapshotHandles) { status = staleWithSnapshot } @@ -252,7 +290,8 @@ func getFileSystem(ctx context.Context, clientsets *k8sutil.Clientsets, operator } // checkSnapshot checks if there are any snapshots in the subvolume -func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, fs, sv, svg string) bool { +// it also check for the stale snapshot and if found, deletes the snapshot. +func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, fs, sv, svg string, snapshotHandles map[string]snapshotInfo) bool { cmd := "ceph" args := []string{"fs", "subvolume", "snapshot", "ls", fs, sv, svg, "--format", "json"} @@ -263,6 +302,19 @@ func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operator return false } snap := unMarshaljson(snapList) + // check for stale subvolume snapshot + // we have the list of snapshothandleid's from the + // volumesnapshotcontent. Looking up for snapid in it + // will confirm if we have stale snapshot or not. + for _, s := range snap { + _, snapId := getSnapOmapVal(s.Name) + // lookup for the snapid in the k8s snapshot handle list + _, ok := snapshotHandles[snapId] + if !ok { + // delete stale snapshot + deleteSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs, sv, svg, s.Name) + } + } if len(snap) == 0 { return false } @@ -296,6 +348,19 @@ func unMarshaljson(list string) []fsStruct { return unmarshal } +// deleteSnapshot deletes the subvolume snapshot +func deleteSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs, subvol, svg, snap string) { + + deleteOmapForSnapshot(ctx, clientsets, OperatorNamespace, CephClusterNamespace, snap, fs) + cmd := "ceph" + args := []string{"fs", "subvolume", "snapshot", "rm", fs, subvol, snap, svg} + + _, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args) + if err != nil { + logging.Fatal(err, "failed to delete subvolume snapshot of %s/%s/%s/%s", fs, svg, subvol, snap) + } +} + func Delete(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs, subvol, svg string) { k8sSubvolume := getK8sRefSubvolume(ctx, clientsets) _, check := k8sSubvolume[subvol] @@ -379,6 +444,41 @@ func deleteOmapForSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets, } } +// deleteOmapForSnapshot deletes omap object and key for the given snapshot. +func deleteOmapForSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, snap, fs string) { + logging.Info("Deleting the omap object and key for snapshot %q", snap) + snapomapkey := getSnapOmapKey(ctx, clientsets, OperatorNamespace, CephClusterNamespace, snap, fs) + snapomapval, _ := getSnapOmapVal(snap) + poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs) + if err != nil || poolName == "" { + logging.Fatal(fmt.Errorf("pool name not found: %q", err)) + } + if snapomapval != "" { + cmd := "rados" + args := []string{"rm", snapomapval, "-p", poolName, "--namespace", "csi"} + + // remove omap object. + _, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args) + if err != nil { + logging.Fatal(err, "failed to remove omap object for snapshot %q", snap) + } + logging.Info("omap object:%q deleted", snapomapval) + + } + if snapomapkey != "" { + cmd := "rados" + args := []string{"rmomapkey", "csi.snaps.default", snapomapkey, "-p", poolName, "--namespace", "csi"} + + // remove omap key. + _, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args) + if err != nil { + logging.Fatal(err, "failed to remove omap key for snapshot %q", snap) + } + logging.Info("omap key:%q deleted", snapomapkey) + + } +} + // getOmapKey gets the omap key and value details for a given subvolume. // Deletion of omap object required the subvolumeName which is of format // csi.volume.subvolume, where subvolume is the name of subvolume that needs to be @@ -407,6 +507,33 @@ func getOmapKey(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNam return omapkey } +// getSnapOmapKey gets the omap key and value details for a given snapshot. +// Deletion of omap object required the snapshotName which is of format +// csi.snap.snapid. +// similarly to delete of omap key requires csi.snap.ompakey, where +// omapkey is the snapshotcontent name which is extracted the omap object. +func getSnapOmapKey(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, snap, fs string) string { + + poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs) + if err != nil || poolName == "" { + logging.Fatal(fmt.Errorf("pool name not found: %q", err)) + } + snapomapval, _ := getSnapOmapVal(snap) + + args := []string{"getomapval", snapomapval, "csi.snapname", "-p", poolName, "--namespace", "csi", "/dev/stdout"} + cmd := "rados" + snapshotcontentname, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args) + if err != nil || snapshotcontentname == "" { + logging.Info("No snapshot content found for snapshot %s: %s", snap, err) + return "" + } + // omap key is for format csi.volume.pvc-fca205e5-8788-4132-979c-e210c0133182 + // hence, attaching pvname to required prefix. + snapomapkey := "csi.snap." + snapshotcontentname + + return snapomapkey +} + // getNfsClusterName returns the cluster name from the omap. // csi.nfs.cluster // value (26 bytes) : @@ -480,3 +607,19 @@ func getOmapVal(subVol string) (string, string) { return omapval, subvolId } + +// func getSnapOmapVal is used to get the omapval from the given snapshot +// omapval is of format csi.snap.427774b4-340b-11ed-8d66-0242ac110005 +// which is similar to volume name csi-snap-427774b4-340b-11ed-8d66-0242ac110005 +// hence, replacing 'csi-vol-' to 'csi.snap.' +func getSnapOmapVal(snap string) (string, string) { + + splitSnap := strings.SplitAfterN(snap, "-", 3) + if len(splitSnap) < 3 { + return "", "" + } + snapId := splitSnap[len(splitSnap)-1] + snapomapval := "csi.snap." + snapId + + return snapomapval, snapId +}