From 2dde0773138df7affcb8c0ba0a4886d1e1fb8e62 Mon Sep 17 00:00:00 2001 From: danfengl Date: Wed, 3 Apr 2024 05:49:31 +0000 Subject: [PATCH] Add checkpoint for FS backup deletion test Signed-off-by: danfengl --- test/e2e/backups/deletion.go | 15 +++- test/e2e/privilegesmgmt/ssr.go | 10 +-- test/util/common/common.go | 3 +- test/util/velero/velero_utils.go | 137 +++++++++++++++++++++++++++---- 4 files changed, 137 insertions(+), 28 deletions(-) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index de710c4cd1..de86f23c34 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -161,13 +161,20 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam } } else { // Check: BackupRepository and DeleteRequest - GetBackupRepository(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.Namespace) - time.Sleep(9 * time.Hour) + var brList, pvbList []string + brList, err = KubectlGetBackupRepository(oneHourTimeout, "kopia", veleroCfg.VeleroNamespace) + pvbList, err = KubectlGetPodVolumeBackup(oneHourTimeout, BackupCfg.BackupName, veleroCfg.VeleroNamespace) + + fmt.Println(brList) + fmt.Println(pvbList) + } - err = DeleteBackupResource(context.Background(), backupName, &veleroCfg) + + err = DeleteBackup(context.Background(), backupName, &veleroCfg) if err != nil { return err } + if useVolumeSnapshots { err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, @@ -214,7 +221,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam return err } - err = DeleteBackupResource(context.Background(), backupName, &veleroCfg) + err = DeleteBackup(context.Background(), backupName, &veleroCfg) if err != nil { return errors.Wrapf(err, "|| UNEXPECTED || - Failed to delete backup %q", backupName) } else { diff --git a/test/e2e/privilegesmgmt/ssr.go b/test/e2e/privilegesmgmt/ssr.go index c5a47da707..d6e955bf54 100644 --- a/test/e2e/privilegesmgmt/ssr.go +++ b/test/e2e/privilegesmgmt/ssr.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/wait" waitutil "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -66,8 +67,8 @@ func SSRTest() { }) ssrListResp := new(v1.ServerStatusRequestList) By(fmt.Sprintf("Check ssr object in %s namespace", veleroCfg.VeleroNamespace)) - err = waitutil.PollImmediate(5*time.Second, time.Minute, - func() (bool, error) { + err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, time.Minute, true, + func(context.Context) (bool, error) { if err = veleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil { return false, fmt.Errorf("failed to list ssr object in %s namespace with err %v", veleroCfg.VeleroNamespace, err) } @@ -85,9 +86,8 @@ func SSRTest() { } return true, nil }) - if err == waitutil.ErrWaitTimeout { - fmt.Printf("exceed test case deadline and failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace) - } + fmt.Printf("exceed test case deadline and failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace)) By(fmt.Sprintf("Check ssr object in %s namespace", testNS)) diff --git a/test/util/common/common.go b/test/util/common/common.go index d014d2a09f..eaf9b6b5d1 100644 --- a/test/util/common/common.go +++ b/test/util/common/common.go @@ -20,10 +20,10 @@ func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string var buf bytes.Buffer var err error var cmds []*exec.Cmd + for _, cmdline := range cmdlines { cmd := exec.Command(cmdline.Cmd, cmdline.Args...) cmds = append(cmds, cmd) - fmt.Println(cmd) } for i := 0; i < len(cmds); i++ { if i == len(cmds)-1 { @@ -55,7 +55,6 @@ func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string if err := scanner.Err(); err != nil { return nil, err } - return ret, nil } diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index fd1bb201e0..859f521863 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -940,7 +940,7 @@ func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) { return tmpfile, nil } -func DeleteBackupResource(ctx context.Context, backupName string, velerocfg *VeleroConfig) error { +func DeleteBackup(ctx context.Context, backupName string, velerocfg *VeleroConfig) error { veleroCLI := velerocfg.VeleroCLI args := []string{"--namespace", velerocfg.VeleroNamespace, "backup", "delete", backupName, "--confirm"} @@ -956,20 +956,46 @@ func DeleteBackupResource(ctx context.Context, backupName string, velerocfg *Vel args = []string{"--namespace", velerocfg.VeleroNamespace, "backup", "get", backupName} - retryTimes := 5 - for i := 1; i < retryTimes+1; i++ { - cmd = exec.CommandContext(ctx, veleroCLI, args...) - fmt.Printf("Try %d times to delete backup %s \n", i, cmd.String()) - stdout, stderr, err = veleroexec.RunCommand(cmd) - if err != nil { - if strings.Contains(stderr, "not found") { - fmt.Printf("|| EXPECTED || - Backup %s was deleted successfully according to message %s\n", backupName, stderr) - return nil + err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, time.Minute, true, + func(context.Context) (bool, error) { + + cmd = exec.CommandContext(ctx, veleroCLI, args...) + fmt.Printf("Try to get backup with cmd: %s \n", cmd.String()) + stdout, stderr, err = veleroexec.RunCommand(cmd) + if err != nil { + if strings.Contains(stderr, "not found") { + fmt.Printf("|| EXPECTED || - Backup %s was deleted successfully according to message %s\n", backupName, stderr) + return true, nil + } + return false, errors.Wrapf(err, "Fail to perform get backup, stdout=%s, stderr=%s", stdout, stderr) } - return errors.Wrapf(err, "Fail to perform get backup, stdout=%s, stderr=%s", stdout, stderr) - } - time.Sleep(1 * time.Minute) + + var status string + var drList []string + drList, err = KubectlGetAllDeleteBackupRequest(context.Background(), backupName, velerocfg.VeleroNamespace) + if len(drList) > 1 { + return false, errors.New(fmt.Sprintf("Count of DeleteBackupRequest %d is not expected", len(drList))) + } + for _, dr := range drList { + status, err = KubectlGetDeleteBackupRequestStatus(context.Background(), dr, velerocfg.VeleroNamespace) + fmt.Printf("DeleteBackupRequest status: %s\n", status) + if err == nil && status == "Processed" { + return true, nil + } else { + return false, nil + } + } + + return true, nil + }) + + time.Sleep(1 * time.Minute) + var drList []string + drList, err = KubectlGetAllDeleteBackupRequest(context.Background(), backupName, velerocfg.VeleroNamespace) + if len(drList) > 1 { + return errors.New(fmt.Sprintf("Count of DeleteBackupRequest %d is not expected", len(drList))) } + return nil } @@ -1655,20 +1681,97 @@ func CleanAllRetainedPV(ctx context.Context, client TestClient) { } } -func GetBackupRepository(ctx context.Context, veleroCLI, veleroNamespace string) ([]string, error) { +func KubectlGetBackupRepository(ctx context.Context, uploaderType, veleroNamespace string) ([]string, error) { args1 := []string{"get", "backuprepository", "-n", veleroNamespace} cmds := []*common.OsCommandLine{} cmd := &common.OsCommandLine{ - Cmd: veleroCLI, + Cmd: "kubectl", + Args: args1, + } + cmds = append(cmds, cmd) + + cmd = &common.OsCommandLine{ + Cmd: "grep", + Args: []string{uploaderType}, + } + cmds = append(cmds, cmd) + + cmd = &common.OsCommandLine{ + Cmd: "awk", + Args: []string{"{print $1}"}, + } + cmds = append(cmds, cmd) + + return common.GetListByCmdPipes(ctx, cmds) +} + +func KubectlGetPodVolumeBackup(ctx context.Context, backupName, veleroNamespace string) ([]string, error) { + args1 := []string{"get", "podvolumebackup", "-n", veleroNamespace} + + cmds := []*common.OsCommandLine{} + + cmd := &common.OsCommandLine{ + Cmd: "kubectl", + Args: args1, + } + cmds = append(cmds, cmd) + + cmd = &common.OsCommandLine{ + Cmd: "grep", + Args: []string{backupName}, + } + cmds = append(cmds, cmd) + + cmd = &common.OsCommandLine{ + Cmd: "awk", + Args: []string{"{print $1}"}, + } + cmds = append(cmds, cmd) + + return common.GetListByCmdPipes(ctx, cmds) +} + +func KubectlGetDeleteBackupRequestStatus(ctx context.Context, deleteBackupRequest, veleroNamespace string) (string, error) { + args1 := []string{"get", "deletebackuprequests", "-n", veleroNamespace, deleteBackupRequest, "-o", "json"} + + cmds := []*common.OsCommandLine{} + + cmd := &common.OsCommandLine{ + Cmd: "kubectl", + Args: args1, + } + cmds = append(cmds, cmd) + + cmd = &common.OsCommandLine{ + Cmd: "jq", + Args: []string{"-r", ".status.phase"}, + } + cmds = append(cmds, cmd) + + ret, err := common.GetListByCmdPipes(ctx, cmds) + + if len(ret) != 1 { + return "", errors.New(fmt.Sprintf("fail to get status of deletebackuprequests %s", deleteBackupRequest)) + } + return ret[0], err +} + +func KubectlGetAllDeleteBackupRequest(ctx context.Context, backupName, veleroNamespace string) ([]string, error) { + args1 := []string{"get", "deletebackuprequests", "-n", veleroNamespace} + + cmds := []*common.OsCommandLine{} + + cmd := &common.OsCommandLine{ + Cmd: "kubectl", Args: args1, } cmds = append(cmds, cmd) cmd = &common.OsCommandLine{ Cmd: "grep", - Args: []string{veleroNamespace}, + Args: []string{backupName}, } cmds = append(cmds, cmd) @@ -1679,4 +1782,4 @@ func GetBackupRepository(ctx context.Context, veleroCLI, veleroNamespace string) cmds = append(cmds, cmd) return common.GetListByCmdPipes(ctx, cmds) -} \ No newline at end of file +}