Skip to content

Commit

Permalink
UPSTREAM: 128972: Add crd from external snapshotter
Browse files Browse the repository at this point in the history
  • Loading branch information
jsafrane committed Jan 2, 2025
1 parent a21f7d6 commit acc737e
Show file tree
Hide file tree
Showing 9 changed files with 94 additions and 111 deletions.
31 changes: 19 additions & 12 deletions test/e2e/storage/framework/volume_group_snapshot_resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
"fmt"

"github.com/onsi/ginkgo/v2"

"github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/kubernetes/test/e2e/framework"
Expand Down Expand Up @@ -56,15 +56,15 @@ type VolumeGroupSnapshotResource struct {
Config *PerTestConfig
Pattern TestPattern

Vgs *unstructured.Unstructured
Vgscontent *unstructured.Unstructured
Vgsclass *unstructured.Unstructured
VGS *unstructured.Unstructured
VGSContent *unstructured.Unstructured
VGSClass *unstructured.Unstructured
}

// CreateVolumeGroupSnapshot creates a VolumeGroupSnapshotClass with given SnapshotDeletionPolicy and a VolumeGroupSnapshot
// from the VolumeGroupSnapshotClass using a dynamic client.
// Returns the unstructured VolumeGroupSnapshotClass and VolumeGroupSnapshot objects.
func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VoulmeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, groupName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) (*unstructured.Unstructured, *unstructured.Unstructured) {
func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VoulmeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, groupName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) (*unstructured.Unstructured, *unstructured.Unstructured, *unstructured.Unstructured) {
defer ginkgo.GinkgoRecover()
var err error
if pattern.SnapshotType != VolumeGroupSnapshot {
Expand Down Expand Up @@ -99,28 +99,35 @@ func CreateVolumeGroupSnapshot(ctx context.Context, sDriver VoulmeGroupSnapshott
ginkgo.By("Getting group snapshot and content")
volumeGroupSnapshot, err = dc.Resource(utils.VolumeGroupSnapshotGVR).Namespace(volumeGroupSnapshot.GetNamespace()).Get(ctx, volumeGroupSnapshot.GetName(), metav1.GetOptions{})
framework.ExpectNoError(err, "Failed to get volume group snapshot after creation")

return gsclass, volumeGroupSnapshot
status := volumeGroupSnapshot.Object["status"]
err = framework.Gomega().Expect(status).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "Failed to get status of volume group snapshot")
vgscName := status.(map[string]interface{})["boundVolumeGroupSnapshotContentName"].(string)
err = framework.Gomega().Expect(vgscName).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "Failed to get content name of volume group snapshot")
vgsc, err := dc.Resource(utils.VolumeGroupSnapshotContentGVR).Get(ctx, vgscName, metav1.GetOptions{})
framework.ExpectNoError(err, "failed to get content of group snapshot")
return gsclass, volumeGroupSnapshot, vgsc
}

// CleanupResource deletes the VolumeGroupSnapshotClass and VolumeGroupSnapshot objects using a dynamic client.
func (r *VolumeGroupSnapshotResource) CleanupResource(ctx context.Context, timeouts *framework.TimeoutContext) error {
defer ginkgo.GinkgoRecover()
dc := r.Config.Framework.DynamicClient
err := dc.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, r.Vgsclass.GetName(), metav1.DeleteOptions{})
err := dc.Resource(utils.VolumeGroupSnapshotClassGVR).Delete(ctx, r.VGSClass.GetName(), metav1.DeleteOptions{})
framework.ExpectNoError(err, "Failed to delete volume group snapshot class")
return nil
}

// CreateVolumeGroupSnapshotResource creates a VolumeGroupSnapshotResource object with the given parameters.
func CreateVolumeGroupSnapshotResource(ctx context.Context, sDriver VoulmeGroupSnapshottableTestDriver, config *PerTestConfig, pattern TestPattern, pvcName string, pvcNamespace string, timeouts *framework.TimeoutContext, parameters map[string]string) *VolumeGroupSnapshotResource {
vgsclass, snapshot := CreateVolumeGroupSnapshot(ctx, sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters)
vgsClass, snapshot, vgsc := CreateVolumeGroupSnapshot(ctx, sDriver, config, pattern, pvcName, pvcNamespace, timeouts, parameters)
vgs := &VolumeGroupSnapshotResource{
Config: config,
Pattern: pattern,
Vgs: snapshot,
Vgsclass: vgsclass,
Vgscontent: nil,
VGS: snapshot,
VGSClass: vgsClass,
VGSContent: vgsc,
}
return vgs
}
31 changes: 23 additions & 8 deletions test/e2e/storage/testsuites/volume_group_snapshottable.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ package testsuites

import (
"context"
"crypto/sha256"
"fmt"

"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
Expand Down Expand Up @@ -176,18 +178,30 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
snapshot := storageframework.CreateVolumeGroupSnapshotResource(ctx, snapshottableDriver, groupTest.config, pattern, labelValue, groupTest.volumeGroup[0][0].Pvc.GetNamespace(), f.Timeouts, map[string]string{"deletionPolicy": pattern.SnapshotDeletionPolicy.String()})
groupTest.snapshots = append(groupTest.snapshots, snapshot)
ginkgo.By("verifying the snapshots in the group are ready to use")
status := snapshot.Vgs.Object["status"]
status := snapshot.VGS.Object["status"]
err := framework.Gomega().Expect(status).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "failed to get status of group snapshot")
volumes := status.(map[string]interface{})["pvcVolumeSnapshotRefList"]
err = framework.Gomega().Expect(volumes).NotTo(gomega.BeNil())

volumeListMap := snapshot.VGSContent.Object["status"].(map[string]interface{})
err = framework.Gomega().Expect(volumeListMap).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "failed to get volume snapshot list")
volumeSnapshotHandlePairList := volumeListMap["volumeSnapshotHandlePairList"].([]interface{})
err = framework.Gomega().Expect(volumeSnapshotHandlePairList).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "failed to get volume snapshot list")
volumeList := volumes.([]interface{})
err = framework.Gomega().Expect(len(volumeList)).To(gomega.Equal(groupTest.numVolumes))
err = framework.Gomega().Expect(len(volumeSnapshotHandlePairList)).To(gomega.Equal(groupTest.numVolumes))
framework.ExpectNoError(err, "failed to get volume snapshot list")
claimSize := groupTest.volumeGroup[0][0].Pvc.Spec.Resources.Requests.Storage().String()
for _, volume := range volumeList {
for _, volume := range volumeSnapshotHandlePairList {
// Create a PVC from the snapshot
volumeHandle := volume.(map[string]interface{})["volumeHandle"].(string)
err = framework.Gomega().Expect(volumeHandle).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "failed to get volume handle from volume")
uid := snapshot.VGSContent.Object["metadata"].(map[string]interface{})["uid"].(string)
err = framework.Gomega().Expect(uid).NotTo(gomega.BeNil())
framework.ExpectNoError(err, "failed to get uuid from content")
volumeSnapshotName := fmt.Sprintf("snapshot-%x", sha256.Sum256([]byte(
uid+volumeHandle)))

pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
StorageClassName: &groupTest.volumeGroup[0][0].Sc.Name,
ClaimSize: claimSize,
Expand All @@ -198,7 +212,7 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
pvc.Spec.DataSource = &v1.TypedLocalObjectReference{
APIGroup: &group,
Kind: "VolumeSnapshot",
Name: volume.(map[string]interface{})["volumeSnapshotRef"].(map[string]interface{})["name"].(string),
Name: volumeSnapshotName,
}

volSrc := v1.VolumeSource{
Expand All @@ -208,8 +222,9 @@ func (s *VolumeGroupSnapshottableTestSuite) DefineTests(driver storageframework.
},
},
}
pvc, err := cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{})
pvc, err = cs.CoreV1().PersistentVolumeClaims(f.Namespace.Name).Create(ctx, pvc, metav1.CreateOptions{})
framework.ExpectNoError(err, "failed to create PVC from snapshot")

pod := StartInPodWithVolumeSource(ctx, cs, volSrc, pvc.Namespace, "snapshot-pod", "sleep 300", groupTest.config.ClientNodeSelection)
ginkgo.DeferCleanup(e2epod.DeletePodWithWait, cs, pod)
framework.ExpectNoError(e2epod.WaitTimeoutForPodRunningInNamespace(ctx, cs, pod.Name, pod.Namespace, f.Timeouts.PodStartSlow), "Pod did not start in expected time")
Expand Down
7 changes: 4 additions & 3 deletions test/e2e/storage/utils/volume_group_snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,16 @@ const (
// VolumeGroupSnapshot is the group snapshot api
VolumeGroupSnapshotAPIGroup = "groupsnapshot.storage.k8s.io"
// VolumeGroupSnapshotAPIVersion is the group snapshot api version
VolumeGroupSnapshotAPIVersion = "groupsnapshot.storage.k8s.io/v1alpha1"
VolumeGroupSnapshotAPIVersion = "groupsnapshot.storage.k8s.io/v1beta1"
)

var (

// VolumeGroupSnapshotGVR is GroupVersionResource for volumegroupsnapshots
VolumeGroupSnapshotGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1alpha1", Resource: "volumegroupsnapshots"}
VolumeGroupSnapshotGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1beta1", Resource: "volumegroupsnapshots"}
// VolumeGroupSnapshotClassGVR is GroupVersionResource for volumegroupsnapshotsclasses
VolumeGroupSnapshotClassGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1alpha1", Resource: "volumegroupsnapshotclasses"}
VolumeGroupSnapshotClassGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1beta1", Resource: "volumegroupsnapshotclasses"}
VolumeGroupSnapshotContentGVR = schema.GroupVersionResource{Group: VolumeGroupSnapshotAPIGroup, Version: "v1beta1", Resource: "volumegroupsnapshotcontents"}
)

// WaitForVolumeGroupSnapshotReady waits for a VolumeGroupSnapshot to be ready to use or until timeout occurs, whichever comes first.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/814"
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150"
controller-gen.kubebuilder.io/version: v0.15.0
name: volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io
spec:
Expand Down Expand Up @@ -31,7 +31,7 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
name: v1beta1
schema:
openAPIV3Schema:
description: |-
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1068"
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150"
controller-gen.kubebuilder.io/version: v0.15.0
name: volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io
spec:
Expand Down Expand Up @@ -53,7 +53,7 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
name: v1beta1
schema:
openAPIV3Schema:
description: |-
Expand Down Expand Up @@ -237,8 +237,6 @@ spec:
- message: both volumeGroupSnapshotRef.name and volumeGroupSnapshotRef.namespace
must be set
rule: has(self.name) && has(self.__namespace__)
- message: volumeGroupSnapshotRef is immutable
rule: self == oldSelf
required:
- deletionPolicy
- driver
Expand All @@ -257,8 +255,9 @@ spec:
The format of this field is a Unix nanoseconds time encoded as an int64.
On Unix, the command date +%s%N returns the current time in nanoseconds
since 1970-01-01 00:00:00 UTC.
format: int64
type: integer
This field is the source for the CreationTime field in VolumeGroupSnapshotStatus
format: date-time
type: string
error:
description: |-
Error is the last observed error during group snapshot creation, if any.
Expand All @@ -276,42 +275,6 @@ spec:
format: date-time
type: string
type: object
pvVolumeSnapshotContentList:
description: |-
PVVolumeSnapshotContentList is the list of pairs of PV and
VolumeSnapshotContent for this group snapshot
The maximum number of allowed snapshots in the group is 100.
items:
description: |-
PVVolumeSnapshotContentPair represent a pair of PV names and
VolumeSnapshotContent names
properties:
persistentVolumeRef:
description: PersistentVolumeRef is a reference to the persistent
volume resource
properties:
name:
description: |-
Name of the referent.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?
type: string
type: object
x-kubernetes-map-type: atomic
volumeSnapshotContentRef:
description: VolumeSnapshotContentRef is a reference to the
volume snapshot content resource
properties:
name:
description: |-
Name of the referent.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?
type: string
type: object
x-kubernetes-map-type: atomic
type: object
type: array
readyToUse:
description: |-
ReadyToUse indicates if all the individual snapshots in the group are ready to be
Expand All @@ -325,6 +288,32 @@ spec:
If a storage system does not provide such an id, the
CSI driver can choose to return the VolumeGroupSnapshot name.
type: string
volumeSnapshotHandlePairList:
description: |-
VolumeSnapshotHandlePairList is a list of CSI "volume_id" and "snapshot_id"
pair returned by the CSI driver to identify snapshots and their source volumes
on the storage system.
items:
description: VolumeSnapshotHandlePair defines a pair of a source
volume handle and a snapshot handle
properties:
snapshotHandle:
description: |-
SnapshotHandle is a unique id returned by the CSI driver to identify a volume
snapshot on the storage system
Required.
type: string
volumeHandle:
description: |-
VolumeHandle is a unique id returned by the CSI driver to identify a volume
on the storage system
Required.
type: string
required:
- snapshotHandle
- volumeHandle
type: object
type: array
type: object
required:
- spec
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1068"
api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/1150"
controller-gen.kubebuilder.io/version: v0.15.0
name: volumegroupsnapshots.groupsnapshot.storage.k8s.io
spec:
Expand Down Expand Up @@ -43,7 +43,7 @@ spec:
- jsonPath: .metadata.creationTimestamp
name: Age
type: date
name: v1alpha1
name: v1beta1
schema:
openAPIV3Schema:
description: |-
Expand Down Expand Up @@ -198,6 +198,7 @@ spec:
The format of this field is a Unix nanoseconds time encoded as an int64.
On Unix, the command date +%s%N returns the current time in nanoseconds
since 1970-01-01 00:00:00 UTC.
This field is updated based on the CreationTime field in VolumeGroupSnapshotContentStatus
format: date-time
type: string
error:
Expand All @@ -221,41 +222,6 @@ spec:
format: date-time
type: string
type: object
pvcVolumeSnapshotRefList:
description: |-
VolumeSnapshotRefList is the list of PVC and VolumeSnapshot pairs that
is part of this group snapshot.
The maximum number of allowed snapshots in the group is 100.
items:
description: PVCVolumeSnapshotPair defines a pair of a PVC reference
and a Volume Snapshot Reference
properties:
persistentVolumeClaimRef:
description: PersistentVolumeClaimRef is a reference to the
PVC this pair is referring to
properties:
name:
description: |-
Name of the referent.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?
type: string
type: object
x-kubernetes-map-type: atomic
volumeSnapshotRef:
description: VolumeSnapshotRef is a reference to the VolumeSnapshot
this pair is referring to
properties:
name:
description: |-
Name of the referent.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?
type: string
type: object
x-kubernetes-map-type: atomic
type: object
type: array
readyToUse:
description: |-
ReadyToUse indicates if all the individual snapshots in the group are ready
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ spec:
serviceAccountName: csi-hostpathplugin-sa
containers:
- name: hostpath
image: registry.k8s.io/sig-storage/hostpathplugin:v1.14.0
image: registry.k8s.io/sig-storage/hostpathplugin:v1.15.0
args:
- "--drivername=hostpath.csi.k8s.io"
- "--v=5"
Expand Down Expand Up @@ -354,11 +354,11 @@ spec:
name: socket-dir

- name: csi-snapshotter
image: registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1
image: registry.k8s.io/sig-storage/csi-snapshotter:v8.2.0
args:
- -v=5
- --csi-address=/csi/csi.sock
- --enable-volume-group-snapshots=true
- --feature-gates=CSIVolumeGroupSnapshot=true
securityContext:
# This is necessary only for systems with SELinux, where
# non-privileged sidecar containers cannot access unix domain socket
Expand Down
Loading

0 comments on commit acc737e

Please sign in to comment.