Skip to content

Commit

Permalink
csi: delete stale subvolumesnapshot
Browse files Browse the repository at this point in the history 
this commit checks if there are any stale
volumesnapshots, if yes, it deletes the
snapshots

Signed-off-by: yati1998 <[email protected]>
  • Loading branch information
@yati1998
yati1998 committed Jul 4, 2024
1 parent a710c73 commit b9b5b61
Show file tree
Hide file tree
Showing 3 changed files with 169 additions and 19 deletions.
11 changes: 7 additions & 4 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,22 @@
module github.com/rook/kubectl-rook-ceph

go 1.21
go 1.22.0

toolchain go1.22.3

require (
github.com/fatih/color v1.17.0
github.com/golang/mock v1.6.0
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0
github.com/pkg/errors v0.9.1
github.com/rook/rook v1.14.7
github.com/rook/rook/pkg/apis v0.0.0-20231204200402-5287527732f7
github.com/spf13/cobra v1.8.1
github.com/stretchr/testify v1.9.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.29.3
k8s.io/apimachinery v0.29.3
k8s.io/client-go v0.29.3
k8s.io/api v0.30.0
k8s.io/apimachinery v0.30.0
k8s.io/client-go v0.30.0
)

require (
Expand Down
24 changes: 14 additions & 10 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1 h1:dQEHhTfi+bSIOSViQrKY9PqJvZenD6tFz+3lPzux58o=
github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1/go.mod h1:my+EVjOJLeQ9lUR9uVkxRvNNkhO2saSGIgzV8GZT9HY=
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0 h1:j3YK74myEQRxR/srciTpOrm221SAvz6J5OVWbyfeXFo=
github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0/go.mod h1:FlyYFe32mPxKEPaRXKNxfX576d1AoCzstYDoOOnyMA4=
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 h1:mjQG0Vakr2h246kEDR85U8y8ZhPgT3bguTCajRa/jaw=
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
github.com/libopenstorage/autopilot-api v0.6.1-0.20210128210103-5fbb67948648/go.mod h1:6JLrPbR3ZJQFbUY/+QJMl/aF00YdIrLf8/GWAplgvJs=
github.com/libopenstorage/openstorage v8.0.0+incompatible/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
github.com/libopenstorage/operator v0.0.0-20200725001727-48d03e197117/go.mod h1:Qh+VXOB6hj60VmlgsmY+R1w+dFuHK246UueM4SAqZG0=
Expand Down Expand Up @@ -605,8 +609,8 @@ github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8Ay
github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo=
github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw=
github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc=
github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4=
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY=
github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
Expand All @@ -620,8 +624,8 @@ github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ
github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=
github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE=
github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk=
github.com/openshift/api v0.0.0-20210105115604-44119421ec6b/go.mod h1:aqU5Cq+kqKKPbDMqxo9FojgDeSpNJI7iuskjXjtojDg=
github.com/openshift/api v0.0.0-20240301093301-ce10821dc999 h1:+S998xHiJApsJZjRAO8wyedU9GfqFd8mtwWly6LqHDo=
github.com/openshift/api v0.0.0-20240301093301-ce10821dc999/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4=
Expand Down Expand Up @@ -1398,8 +1402,8 @@ k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8=
k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg=
k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw=
k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80=
k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA=
k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE=
k8s.io/apiextensions-apiserver v0.0.0-20190409022649-727a075fdec8/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE=
k8s.io/apiextensions-apiserver v0.18.3/go.mod h1:TMsNGs7DYpMXd+8MOCX8KzPOCx8fnZMoIGB24m03+JE=
k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk=
Expand All @@ -1412,8 +1416,8 @@ k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRp
k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM=
k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74=
k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU=
k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU=
k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA=
k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
k8s.io/apiserver v0.18.3/go.mod h1:tHQRmthRPLUtwqsOnJJMoI8SW3lnoReZeE861lH8vUw=
k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw=
Expand All @@ -1422,8 +1426,8 @@ k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA=
k8s.io/client-go v0.20.0/go.mod h1:4KWh/g+Ocd8KkCwKF8vUNnmqgv+EVnQDK4MBF4oB5tY=
k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4=
k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg=
k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0=
k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ=
k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY=
k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk=
k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg=
Expand Down
153 changes: 148 additions & 5 deletions pkg/filesystem/subvolume.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"fmt"
"strings"

snapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1"
"github.com/rook/kubectl-rook-ceph/pkg/exec"
"github.com/rook/kubectl-rook-ceph/pkg/k8sutil"
"github.com/rook/kubectl-rook-ceph/pkg/logging"
Expand All @@ -40,6 +41,11 @@ type subVolumeInfo struct {
state string
}

type snapshotInfo struct {
volumehandle string
snapshothandle string
}

type monitor struct {
ClusterID string
Monitors []string
Expand All @@ -49,12 +55,14 @@ const (
inUse = "in-use"
stale = "stale"
staleWithSnapshot = "stale-with-snapshot"
snapshotRetained = "snapshot-retained"
)

func List(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool) {

subvolumeNames := getK8sRefSubvolume(ctx, clientsets)
listCephFSSubvolumes(ctx, clientsets, operatorNamespace, clusterNamespace, includeStaleOnly, subvolumeNames)
snapshotHandles := getK8sRefSnapshotHandle(ctx, clientsets)
listCephFSSubvolumes(ctx, clientsets, operatorNamespace, clusterNamespace, includeStaleOnly, subvolumeNames, snapshotHandles)
}

// checkForExternalStorage checks if the external mode is enabled.
Expand Down Expand Up @@ -131,6 +139,36 @@ func getK8sRefSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets) map
return subvolumeNames
}

// getk8sRefSnapshotHandle returns the snapshothandle for k8s ref of the volume snapshots
func getK8sRefSnapshotHandle(ctx context.Context, clientsets *k8sutil.Clientsets) map[string]snapshotInfo {

Snap, err := snapclient.NewForConfig(clientsets.KubeConfig)
if err != nil {
logging.Fatal(err)
}
snapList, err := Snap.VolumeSnapshotContents().List(ctx, v1.ListOptions{})
if err != nil {
logging.Fatal(fmt.Errorf("Error fetching volumesnapshotcontents: %v\n", err))
}

snapshotHandles := make(map[string]snapshotInfo)
for _, snap := range snapList.Items {
if snap.Status != nil && snap.Status.SnapshotHandle != nil {
// get the snaps id from snapshot handle
splitSnapshotHandle := strings.SplitAfterN(*snap.Status.SnapshotHandle, "-", 6)
if len(splitSnapshotHandle) < 6 {
return nil
}
snapshotHandleId := splitSnapshotHandle[len(splitSnapshotHandle)-1]
// map the volumeHandle id to later lookup for the subvol id and
// match the subvolume snapshot.
snapshotHandles[snapshotHandleId] = snapshotInfo{}
}
}

return snapshotHandles
}

// runCommand checks for the presence of externalcluster and runs the command accordingly.
func runCommand(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, cmd string, args []string) (string, error) {
if checkForExternalStorage(ctx, clientsets, clusterNamespace) {
Expand All @@ -143,7 +181,7 @@ func runCommand(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNam
}

// listCephFSSubvolumes list all the subvolumes
func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool, subvolumeNames map[string]subVolumeInfo) {
func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool, subvolumeNames map[string]subVolumeInfo, snapshotHandles map[string]snapshotInfo) {

// getFilesystem gets the filesystem
fsstruct, err := getFileSystem(ctx, clientsets, operatorNamespace, clusterNamespace)
Expand Down Expand Up @@ -195,11 +233,11 @@ func listCephFSSubvolumes(ctx context.Context, clientsets *k8sutil.Clientsets, o
// check the state of the stale subvolume
// if it is snapshot-retained then skip listing it.
if state == "snapshot-retained" {
status = state
status = snapshotRetained
continue
}
// check if the stale subvolume has snapshots.
if checkSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs.Name, sv.Name, svg.Name) {
if checkSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs.Name, sv.Name, svg.Name, snapshotHandles) {
status = staleWithSnapshot
}

Expand Down Expand Up @@ -252,7 +290,8 @@ func getFileSystem(ctx context.Context, clientsets *k8sutil.Clientsets, operator
}

// checkSnapshot checks if there are any snapshots in the subvolume
func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, fs, sv, svg string) bool {
// it also check for the stale snapshot and if found, deletes the snapshot.
func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace, fs, sv, svg string, snapshotHandles map[string]snapshotInfo) bool {

cmd := "ceph"
args := []string{"fs", "subvolume", "snapshot", "ls", fs, sv, svg, "--format", "json"}
Expand All @@ -263,6 +302,19 @@ func checkSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, operator
return false
}
snap := unMarshaljson(snapList)
// check for stale subvolume snapshot
// we have the list of snapshothandleid's from the
// volumesnapshotcontent. Looking up for snapid in it
// will confirm if we have stale snapshot or not.
for _, s := range snap {
snapId, _ := getSnapOmapVal(s.Name)
// lookup for the snapid in the k8s snapshot handle list
_, ok := snapshotHandles[snapId]
if !ok {
// delete stale snapshot
deleteSnapshot(ctx, clientsets, operatorNamespace, clusterNamespace, fs, sv, svg, s.Name)
}
}
if len(snap) == 0 {
return false
}
Expand Down Expand Up @@ -296,6 +348,19 @@ func unMarshaljson(list string) []fsStruct {
return unmarshal
}

// deleteSnapshot deletes the subvolume snapshot
func deleteSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs, subvol, svg, snap string) {

deleteOmapForSnapshot(ctx, clientsets, OperatorNamespace, CephClusterNamespace, snap, fs)
cmd := "ceph"
args := []string{"fs", "subvolume", "snapshot", "rm", fs, subvol, snap, svg}

_, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args)
if err != nil {
logging.Fatal(err, "failed to delete subvolume snapshot of %s/%s/%s/%s", fs, svg, subvol, snap)
}
}

func Delete(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs, subvol, svg string) {
k8sSubvolume := getK8sRefSubvolume(ctx, clientsets)
_, check := k8sSubvolume[subvol]
Expand Down Expand Up @@ -379,6 +444,41 @@ func deleteOmapForSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets,
}
}

// deleteOmapForSnapshot deletes omap object and key for the given snapshot.
func deleteOmapForSnapshot(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, snap, fs string) {
logging.Info("Deleting the omap object and key for snapshot %q", snap)
snapomapkey := getSnapOmapKey(ctx, clientsets, OperatorNamespace, CephClusterNamespace, snap, fs)
snapomapval, _ := getSnapOmapVal(snap)
poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs)
if err != nil || poolName == "" {
logging.Fatal(fmt.Errorf("pool name not found: %q", err))
}
if snapomapval != "" {
cmd := "rados"
args := []string{"rm", snapomapval, "-p", poolName, "--namespace", "csi"}

// remove omap object.
_, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args)
if err != nil {
logging.Fatal(err, "failed to remove omap object for snapshot %q", snap)
}
logging.Info("omap object:%q deleted", snapomapval)

}
if snapomapkey != "" {
cmd := "rados"
args := []string{"rmomapkey", "csi.snaps.default", snapomapkey, "-p", poolName, "--namespace", "csi"}

// remove omap key.
_, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args)
if err != nil {
logging.Fatal(err, "failed to remove omap key for snapshot %q", snap)
}
logging.Info("omap key:%q deleted", snapomapkey)

}
}

// getOmapKey gets the omap key and value details for a given subvolume.
// Deletion of omap object required the subvolumeName which is of format
// csi.volume.subvolume, where subvolume is the name of subvolume that needs to be
Expand Down Expand Up @@ -407,6 +507,33 @@ func getOmapKey(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNam
return omapkey
}

// getSnapOmapKey gets the omap key and value details for a given snapshot.
// Deletion of omap object required the snapshotName which is of format
// csi.snap.snapid.
// similarly to delete of omap key requires csi.snap.ompakey, where
// omapkey is the snapshotcontent name which is extracted the omap object.
func getSnapOmapKey(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, snap, fs string) string {

poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs)
if err != nil || poolName == "" {
logging.Fatal(fmt.Errorf("pool name not found: %q", err))
}
snapomapval, _ := getSnapOmapVal(snap)

args := []string{"getomapval", snapomapval, "csi.snapname", "-p", poolName, "--namespace", "csi", "/dev/stdout"}
cmd := "rados"
snapshotcontentname, err := runCommand(ctx, clientsets, OperatorNamespace, CephClusterNamespace, cmd, args)
if err != nil || snapshotcontentname == "" {
logging.Info("No snapshot content found for snapshot %s: %s", snap, err)
return ""
}
// omap key is for format csi.volume.pvc-fca205e5-8788-4132-979c-e210c0133182
// hence, attaching pvname to required prefix.
snapomapkey := "csi.snap." + snapshotcontentname

return snapomapkey
}

// getNfsClusterName returns the cluster name from the omap.
// csi.nfs.cluster
// value (26 bytes) :
Expand Down Expand Up @@ -480,3 +607,19 @@ func getOmapVal(subVol string) (string, string) {

return omapval, subvolId
}

// func getSnapOmapVal is used to get the omapval from the given snapshot
// omapval is of format csi.snap.427774b4-340b-11ed-8d66-0242ac110005
// which is similar to volume name csi-snap-427774b4-340b-11ed-8d66-0242ac110005
// hence, replacing 'csi-vol-' to 'csi.snap.'
func getSnapOmapVal(snap string) (string, string) {

splitSnap := strings.SplitAfterN(snap, "-", 3)
if len(splitSnap) < 3 {
return "", ""
}
snapId := splitSnap[len(splitSnap)-1]
snapomapval := "csi.snap." + snapId

return snapomapval, snapId
}

0 comments on commit b9b5b61

Please sign in to comment.