Add checkpoint for FS backup deletion test

Signed-off-by: danfengl <[email protected]>
vmware-tanzu · Apr 3, 2024 · 2dde077 · 2dde077
1 parent 988c2fa
commit 2dde077
Show file tree

Hide file tree

Showing 4 changed files with 137 additions and 28 deletions.
diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go
@@ -161,13 +161,20 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		}
 	} else {
 		// Check: BackupRepository and DeleteRequest
-		GetBackupRepository(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.Namespace)
-		time.Sleep(9 * time.Hour)
+		var brList, pvbList []string
+		brList, err = KubectlGetBackupRepository(oneHourTimeout, "kopia", veleroCfg.VeleroNamespace)
+		pvbList, err = KubectlGetPodVolumeBackup(oneHourTimeout, BackupCfg.BackupName, veleroCfg.VeleroNamespace)
+
+		fmt.Println(brList)
+		fmt.Println(pvbList)
+
 	}
-	err = DeleteBackupResource(context.Background(), backupName, &veleroCfg)
+
+	err = DeleteBackup(context.Background(), backupName, &veleroCfg)
 	if err != nil {
 		return err
 	}
+
 	if useVolumeSnapshots {
 		err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
 			veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
@@ -214,7 +221,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		return err
 	}
 
-	err = DeleteBackupResource(context.Background(), backupName, &veleroCfg)
+	err = DeleteBackup(context.Background(), backupName, &veleroCfg)
 	if err != nil {
 		return errors.Wrapf(err, "|| UNEXPECTED || - Failed to delete backup %q", backupName)
 	} else {

diff --git a/test/e2e/privilegesmgmt/ssr.go b/test/e2e/privilegesmgmt/ssr.go
@@ -25,6 +25,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
 	waitutil "k8s.io/apimachinery/pkg/util/wait"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -66,8 +67,8 @@ func SSRTest() {
 		})
 		ssrListResp := new(v1.ServerStatusRequestList)
 		By(fmt.Sprintf("Check ssr object in %s namespace", veleroCfg.VeleroNamespace))
-		err = waitutil.PollImmediate(5*time.Second, time.Minute,
-			func() (bool, error) {
+		err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, time.Minute, true,
+			func(context.Context) (bool, error) {
 				if err = veleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil {
 					return false, fmt.Errorf("failed to list ssr object in %s namespace with err %v", veleroCfg.VeleroNamespace, err)
 				}
@@ -85,9 +86,8 @@ func SSRTest() {
 				}
 				return true, nil
 			})
-		if err == waitutil.ErrWaitTimeout {
-			fmt.Printf("exceed test case deadline and failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace)
-		}
+		fmt.Printf("exceed test case deadline and failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace)
+
 		Expect(err).To(Succeed(), fmt.Sprintf("Failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace))
 
 		By(fmt.Sprintf("Check ssr object in %s namespace", testNS))

diff --git a/test/util/common/common.go b/test/util/common/common.go
@@ -20,10 +20,10 @@ func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string
 	var buf bytes.Buffer
 	var err error
 	var cmds []*exec.Cmd
+
 	for _, cmdline := range cmdlines {
 		cmd := exec.Command(cmdline.Cmd, cmdline.Args...)
 		cmds = append(cmds, cmd)
-		fmt.Println(cmd)
 	}
 	for i := 0; i < len(cmds); i++ {
 		if i == len(cmds)-1 {
@@ -55,7 +55,6 @@ func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string
 	if err := scanner.Err(); err != nil {
 		return nil, err
 	}
-
 	return ret, nil
 }
 

diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go
@@ -940,7 +940,7 @@ func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) {
 	return tmpfile, nil
 }
 
-func DeleteBackupResource(ctx context.Context, backupName string, velerocfg *VeleroConfig) error {
+func DeleteBackup(ctx context.Context, backupName string, velerocfg *VeleroConfig) error {
 	veleroCLI := velerocfg.VeleroCLI
 	args := []string{"--namespace", velerocfg.VeleroNamespace, "backup", "delete", backupName, "--confirm"}
 
@@ -956,20 +956,46 @@ func DeleteBackupResource(ctx context.Context, backupName string, velerocfg *Vel
 
 	args = []string{"--namespace", velerocfg.VeleroNamespace, "backup", "get", backupName}
 
-	retryTimes := 5
-	for i := 1; i < retryTimes+1; i++ {
-		cmd = exec.CommandContext(ctx, veleroCLI, args...)
-		fmt.Printf("Try %d times to delete backup %s \n", i, cmd.String())
-		stdout, stderr, err = veleroexec.RunCommand(cmd)
-		if err != nil {
-			if strings.Contains(stderr, "not found") {
-				fmt.Printf("|| EXPECTED || - Backup %s was deleted successfully according to message %s\n", backupName, stderr)
-				return nil
+	err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, time.Minute, true,
+		func(context.Context) (bool, error) {
+
+			cmd = exec.CommandContext(ctx, veleroCLI, args...)
+			fmt.Printf("Try to get backup with cmd: %s \n", cmd.String())
+			stdout, stderr, err = veleroexec.RunCommand(cmd)
+			if err != nil {
+				if strings.Contains(stderr, "not found") {
+					fmt.Printf("|| EXPECTED || - Backup %s was deleted successfully according to message %s\n", backupName, stderr)
+					return true, nil
+				}
+				return false, errors.Wrapf(err, "Fail to perform get backup, stdout=%s, stderr=%s", stdout, stderr)
 			}
-			return errors.Wrapf(err, "Fail to perform get backup, stdout=%s, stderr=%s", stdout, stderr)
-		}
-		time.Sleep(1 * time.Minute)
+
+			var status string
+			var drList []string
+			drList, err = KubectlGetAllDeleteBackupRequest(context.Background(), backupName, velerocfg.VeleroNamespace)
+			if len(drList) > 1 {
+				return false, errors.New(fmt.Sprintf("Count of DeleteBackupRequest %d is not expected", len(drList)))
+			}
+			for _, dr := range drList {
+				status, err = KubectlGetDeleteBackupRequestStatus(context.Background(), dr, velerocfg.VeleroNamespace)
+				fmt.Printf("DeleteBackupRequest status: %s\n", status)
+				if err == nil && status == "Processed" {
+					return true, nil
+				} else {
+					return false, nil
+				}
+			}
+
+			return true, nil
+		})
+
+	time.Sleep(1 * time.Minute)
+	var drList []string
+	drList, err = KubectlGetAllDeleteBackupRequest(context.Background(), backupName, velerocfg.VeleroNamespace)
+	if len(drList) > 1 {
+		return errors.New(fmt.Sprintf("Count of DeleteBackupRequest %d is not expected", len(drList)))
 	}
+
 	return nil
 }
 
@@ -1655,20 +1681,97 @@ func CleanAllRetainedPV(ctx context.Context, client TestClient) {
 	}
 }
 
-func GetBackupRepository(ctx context.Context, veleroCLI, veleroNamespace string) ([]string, error) {
+func KubectlGetBackupRepository(ctx context.Context, uploaderType, veleroNamespace string) ([]string, error) {
 	args1 := []string{"get", "backuprepository", "-n", veleroNamespace}
 
 	cmds := []*common.OsCommandLine{}
 
 	cmd := &common.OsCommandLine{
-		Cmd:  veleroCLI,
+		Cmd:  "kubectl",
+		Args: args1,
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "grep",
+		Args: []string{uploaderType},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "awk",
+		Args: []string{"{print $1}"},
+	}
+	cmds = append(cmds, cmd)
+
+	return common.GetListByCmdPipes(ctx, cmds)
+}
+
+func KubectlGetPodVolumeBackup(ctx context.Context, backupName, veleroNamespace string) ([]string, error) {
+	args1 := []string{"get", "podvolumebackup", "-n", veleroNamespace}
+
+	cmds := []*common.OsCommandLine{}
+
+	cmd := &common.OsCommandLine{
+		Cmd:  "kubectl",
+		Args: args1,
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "grep",
+		Args: []string{backupName},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "awk",
+		Args: []string{"{print $1}"},
+	}
+	cmds = append(cmds, cmd)
+
+	return common.GetListByCmdPipes(ctx, cmds)
+}
+
+func KubectlGetDeleteBackupRequestStatus(ctx context.Context, deleteBackupRequest, veleroNamespace string) (string, error) {
+	args1 := []string{"get", "deletebackuprequests", "-n", veleroNamespace, deleteBackupRequest, "-o", "json"}
+
+	cmds := []*common.OsCommandLine{}
+
+	cmd := &common.OsCommandLine{
+		Cmd:  "kubectl",
+		Args: args1,
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &common.OsCommandLine{
+		Cmd:  "jq",
+		Args: []string{"-r", ".status.phase"},
+	}
+	cmds = append(cmds, cmd)
+
+	ret, err := common.GetListByCmdPipes(ctx, cmds)
+
+	if len(ret) != 1 {
+		return "", errors.New(fmt.Sprintf("fail to get status of deletebackuprequests %s", deleteBackupRequest))
+	}
+	return ret[0], err
+}
+
+func KubectlGetAllDeleteBackupRequest(ctx context.Context, backupName, veleroNamespace string) ([]string, error) {
+	args1 := []string{"get", "deletebackuprequests", "-n", veleroNamespace}
+
+	cmds := []*common.OsCommandLine{}
+
+	cmd := &common.OsCommandLine{
+		Cmd:  "kubectl",
 		Args: args1,
 	}
 	cmds = append(cmds, cmd)
 
 	cmd = &common.OsCommandLine{
 		Cmd:  "grep",
-		Args: []string{veleroNamespace},
+		Args: []string{backupName},
 	}
 	cmds = append(cmds, cmd)
 
@@ -1679,4 +1782,4 @@ func GetBackupRepository(ctx context.Context, veleroCLI, veleroNamespace string)
 	cmds = append(cmds, cmd)
 
 	return common.GetListByCmdPipes(ctx, cmds)
-}
+}