Skip to content

Commit

Permalink
Add checkpoint for FS backup deletion test
Browse files Browse the repository at this point in the history 
As per PR #7281, if repository count is more than 1, then snapshots deletion is achieved with a fast way, then we should have more than 1 FS backup repository per backup.

Signed-off-by: danfengl <[email protected]>
  • Loading branch information
@danfengliu
danfengliu committed May 7, 2024
1 parent a798182 commit 45c9999
Show file tree
Hide file tree
Showing 4 changed files with 251 additions and 65 deletions.
131 changes: 85 additions & 46 deletions test/e2e/backups/deletion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"context"
"flag"
"fmt"
"strings"
"time"

"github.com/google/uuid"
Expand All @@ -33,8 +34,6 @@ import (
. "github.com/vmware-tanzu/velero/test/util/velero"
)

const deletionTest = "deletion-workload"

// Test backup and restore of Kibishi using restic

func BackupDeletionWithSnapshots() {
Expand All @@ -45,11 +44,7 @@ func BackupDeletionWithRestic() {
backup_deletion_test(false)
}
func backup_deletion_test(useVolumeSnapshots bool) {
var (
backupName string
veleroCfg VeleroConfig
)
veleroCfg = VeleroCfg
veleroCfg := VeleroCfg
veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
veleroCfg.UseNodeAgent = !useVolumeSnapshots

Expand All @@ -76,16 +71,23 @@ func backup_deletion_test(useVolumeSnapshots bool) {

When("kibishii is the sample workload", func() {
It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() {
backupName = "backup-" + UUIDgen.String()
Expect(runBackupDeletionTests(*veleroCfg.ClientToInstallVelero, veleroCfg, backupName, "", useVolumeSnapshots, veleroCfg.KibishiiDirectory)).To(Succeed(),
Expect(runBackupDeletionTests(*veleroCfg.ClientToInstallVelero, veleroCfg, "", useVolumeSnapshots, veleroCfg.KibishiiDirectory)).To(Succeed(),
"Failed to run backup deletion test")
})
})
}

// runBackupDeletionTests runs upgrade test on the provider by kibishii.
func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupName, backupLocation string,
// runUpgradeTests runs upgrade test on the provider by kibishii.
func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLocation string,
useVolumeSnapshots bool, kibishiiDirectory string) error {
var err error
var snapshotCheckPoint SnapshotCheckPoint
backupName := "backup-" + UUIDgen.String()

workloadNamespaceList := []string{"backup-deletion-1-" + UUIDgen.String(), "backup-deletion-2-" + UUIDgen.String()}
nsCount := len(workloadNamespaceList)
workloadNamespaces := strings.Join(workloadNamespaceList[:], ",")

if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
Expand All @@ -98,29 +100,30 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
bslPrefix := veleroCfg.BSLPrefix
bslConfig := veleroCfg.BSLConfig
veleroFeatures := veleroCfg.Features

if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest)
}
if !veleroCfg.Debug {
defer func() {
if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest))
}
}()
for _, ns := range workloadNamespaceList {
if err := CreateNamespace(oneHourTimeout, client, ns); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", ns)
}
if !veleroCfg.Debug {
defer func() {
if err := DeleteNamespace(context.Background(), client, ns, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns))
}
}()
}
if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, ns,
registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil {
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", ns)
}
err := ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
if err != nil {
return err
}
}

if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest,
registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil {
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest)
}
err := ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
if err != nil {
return err
}
var BackupCfg BackupConfig
BackupCfg.BackupName = backupName
BackupCfg.Namespace = deletionTest
BackupCfg.Namespace = workloadNamespaces
BackupCfg.BackupLocation = backupLocation
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
Expand All @@ -133,34 +136,70 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
return "Fail to backup workload"
})
})

if providerName == Vsphere && useVolumeSnapshots {
// Wait for uploads started by the Velero Plugin for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
fmt.Println("Waiting for vSphere uploads to complete")
if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, deletionTest, 2); err != nil {
return errors.Wrapf(err, "Error waiting for uploads to complete")
for _, ns := range workloadNamespaceList {
if providerName == Vsphere && useVolumeSnapshots {
// Wait for uploads started by the Velero Plugin for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
fmt.Println("Waiting for vSphere uploads to complete")
if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, ns, DefaultKibishiiWorkerCounts); err != nil {
return errors.Wrapf(err, "Error waiting for uploads to complete")
}
}
}
err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
if err != nil {
return err
}
var snapshotCheckPoint SnapshotCheckPoint

if useVolumeSnapshots {
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, deletionTest, backupName, KibishiiPVCNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig,
backupName, snapshotCheckPoint)
// Check for snapshots existence
if veleroCfg.CloudProvider == Vsphere {
// For vSphere, checking snapshot should base on namespace and backup name
for _, ns := range workloadNamespaceList {
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, DefaultKibishiiWorkerCounts, ns, backupName, KibishiiPVCNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
}
} else {
// For public cloud, When using backup name to index VolumeSnapshotContents, make sure count of VolumeSnapshotContents should including PVs in all namespace
// so VolumeSnapshotContents count should be equal to "namespace count" * "Kibishii worker count per namespace".
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, DefaultKibishiiWorkerCounts*nsCount, "", backupName, KibishiiPVCNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")

// Get all snapshots base on backup name, regardless of namespaces
err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
}
} else {
// Check for BackupRepository and DeleteRequest
var brList, pvbList []string
brList, err = KubectlGetBackupRepository(oneHourTimeout, "kopia", veleroCfg.VeleroNamespace)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
return err
}
pvbList, err = KubectlGetPodVolumeBackup(oneHourTimeout, BackupCfg.BackupName, veleroCfg.VeleroNamespace)

fmt.Println(brList)
fmt.Println(pvbList)
if err != nil {
return err
}
}
err = DeleteBackupResource(context.Background(), backupName, &veleroCfg)

err = DeleteBackup(context.Background(), backupName, &veleroCfg)
if err != nil {
return err
}

if useVolumeSnapshots {
err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
Expand Down Expand Up @@ -207,7 +246,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
return err
}

err = DeleteBackupResource(context.Background(), backupName, &veleroCfg)
err = DeleteBackup(context.Background(), backupName, &veleroCfg)
if err != nil {
return errors.Wrapf(err, "|| UNEXPECTED || - Failed to delete backup %q", backupName)
} else {
Expand Down
10 changes: 5 additions & 5 deletions test/e2e/privilegesmgmt/ssr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/wait"
waitutil "k8s.io/apimachinery/pkg/util/wait"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"

Expand Down Expand Up @@ -66,8 +67,8 @@ func SSRTest() {
})
ssrListResp := new(v1.ServerStatusRequestList)
By(fmt.Sprintf("Check ssr object in %s namespace", veleroCfg.VeleroNamespace))
err = waitutil.PollImmediate(5*time.Second, time.Minute,
func() (bool, error) {
err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, time.Minute, true,
func(context.Context) (bool, error) {
if err = veleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil {
return false, fmt.Errorf("failed to list ssr object in %s namespace with err %v", veleroCfg.VeleroNamespace, err)
}
Expand All @@ -85,9 +86,8 @@ func SSRTest() {
}
return true, nil
})
if err == waitutil.ErrWaitTimeout {
fmt.Printf("exceed test case deadline and failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace)
}
fmt.Printf("exceed test case deadline and failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace)

Expect(err).To(Succeed(), fmt.Sprintf("Failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace))

By(fmt.Sprintf("Check ssr object in %s namespace", testNS))
Expand Down
4 changes: 2 additions & 2 deletions test/util/common/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,12 @@ func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string
var buf bytes.Buffer
var err error
var cmds []*exec.Cmd

for _, cmdline := range cmdlines {
cmd := exec.Command(cmdline.Cmd, cmdline.Args...)
cmds = append(cmds, cmd)
fmt.Println(cmd)
}
fmt.Println(cmds)
for i := 0; i < len(cmds); i++ {
if i == len(cmds)-1 {
break
Expand Down Expand Up @@ -55,7 +56,6 @@ func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string
if err := scanner.Err(); err != nil {
return nil, err
}

return ret, nil
}

Expand Down
Loading

0 comments on commit 45c9999

Please sign in to comment.