feat(restic): support for using a pvc as a repository

Signed-off-by: Devin Buhl <[email protected]> fix: run make bundle Signed-off-by: Devin Buhl <[email protected]> Initial work on test (#1) Co-authored-by: Devin Buhl <[email protected]> fix: run make custom-scorecard-tests-generate-config Signed-off-by: Devin Buhl <[email protected]> Update test-e2e/test_restic_manual_PVC_copy_trigger.yml Signed-off-by: Devin Buhl <[email protected]> fix: update linting issues Signed-off-by: Devin Buhl <[email protected]> fix: update e2e test Signed-off-by: Devin Buhl <[email protected]> Rename test_restic_manual_PVC_copy_trigger.yml to test_restic_manual_pvc_copy_trigger.yml Signed-off-by: Devin Buhl <[email protected]> fix: update e2e test Signed-off-by: Devin Buhl <[email protected]> add vars to restic builder Signed-off-by: Devin Buhl <[email protected]>
backube · Oct 18, 2024 · 1fdf146 · 1fdf146
1 parent 493d997
commit 1fdf146
Show file tree

Hide file tree

Showing 16 changed files with 419 additions and 7 deletions.
diff --git a/api/v1alpha1/replicationdestination_types.go b/api/v1alpha1/replicationdestination_types.go
@@ -260,6 +260,9 @@ type ReplicationDestinationResticSpec struct {
 	// Defaults to false.
 	//+optional
 	EnableFileDeletion bool `json:"enableFileDeletion,omitempty"`
+	// repositoryPVC is the name of an existing PVC containing the backup repository
+	//+optional
+	RepositoryPVC string `json:"repositoryPVC,omitempty"`
 
 	MoverConfig `json:",inline"`
 }

diff --git a/api/v1alpha1/replicationsource_types.go b/api/v1alpha1/replicationsource_types.go
@@ -209,6 +209,9 @@ type ReplicationSourceResticSpec struct {
 	// then ran a backup.
 	// Unlock will not be run again unless spec.restic.unlock is set to a different value.
 	Unlock string `json:"unlock,omitempty"`
+	// repositoryPVC is the name of an existing PVC containing the backup repository
+	//+optional
+	RepositoryPVC string `json:"repositoryPVC,omitempty"`
 
 	MoverConfig `json:",inline"`
 }

diff --git a/bundle/manifests/volsync.backube_replicationdestinations.yaml b/bundle/manifests/volsync.backube_replicationdestinations.yaml
@@ -2703,6 +2703,10 @@ spec:
                     description: Repository is the secret name containing repository
                       info
                     type: string
+                  repositoryPVC:
+                    description: repositoryPVC is the name of an existing PVC containing
+                      the backup repository
+                    type: string
                   restoreAsOf:
                     description: RestoreAsOf refers to the backup that is most recent
                       as of that time.

diff --git a/bundle/manifests/volsync.backube_replicationsources.yaml b/bundle/manifests/volsync.backube_replicationsources.yaml
@@ -2653,6 +2653,10 @@ spec:
                     description: Repository is the secret name containing repository
                       info
                     type: string
+                  repositoryPVC:
+                    description: repositoryPVC is the name of an existing PVC containing
+                      the backup repository
+                    type: string
                   retain:
                     description: ResticRetainPolicy define the retain policy
                     properties:

diff --git a/bundle/manifests/volsync.clusterserviceversion.yaml b/bundle/manifests/volsync.clusterserviceversion.yaml
@@ -55,7 +55,7 @@ metadata:
         }
       ]
     capabilities: Basic Install
-    createdAt: "2024-09-30T20:33:48Z"
+    createdAt: "2024-10-18T13:01:48Z"
     olm.skipRange: '>=0.4.0 <0.11.0'
     operators.operatorframework.io/builder: operator-sdk-v1.31.0
     operators.operatorframework.io/project_layout: go.kubebuilder.io/v3

diff --git a/config/crd/bases/volsync.backube_replicationdestinations.yaml b/config/crd/bases/volsync.backube_replicationdestinations.yaml
@@ -2703,6 +2703,10 @@ spec:
                     description: Repository is the secret name containing repository
                       info
                     type: string
+                  repositoryPVC:
+                    description: repositoryPVC is the name of an existing PVC containing
+                      the backup repository
+                    type: string
                   restoreAsOf:
                     description: RestoreAsOf refers to the backup that is most recent
                       as of that time.

diff --git a/config/crd/bases/volsync.backube_replicationsources.yaml b/config/crd/bases/volsync.backube_replicationsources.yaml
@@ -2653,6 +2653,10 @@ spec:
                     description: Repository is the secret name containing repository
                       info
                     type: string
+                  repositoryPVC:
+                    description: repositoryPVC is the name of an existing PVC containing
+                      the backup repository
+                    type: string
                   retain:
                     description: ResticRetainPolicy define the retain policy
                     properties:

diff --git a/controllers/mover/restic/builder.go b/controllers/mover/restic/builder.go
@@ -147,6 +147,7 @@ func (rb *Builder) FromSource(client client.Client, logger logr.Logger,
 		sourceStatus:          source.Status.Restic,
 		latestMoverStatus:     source.Status.LatestMoverStatus,
 		moverConfig:           source.Spec.Restic.MoverConfig,
+		repositoryPVC:         source.Spec.Restic.RepositoryPVC,
 	}, nil
 }
 
@@ -201,5 +202,6 @@ func (rb *Builder) FromDestination(client client.Client, logger logr.Logger,
 		enableFileDeletionOnRestore: destination.Spec.Restic.EnableFileDeletion,
 		latestMoverStatus:           destination.Status.LatestMoverStatus,
 		moverConfig:                 destination.Spec.Restic.MoverConfig,
+		repositoryPVC:               destination.Spec.Restic.RepositoryPVC,
 	}, nil
 }
diff --git a/controllers/mover/restic/mover.go b/controllers/mover/restic/mover.go
@@ -54,6 +54,8 @@ const (
 	resticCAFilename     = "ca.crt"
 	credentialDir        = "/credentials"
 	gcsCredentialFile    = "gcs.json"
+	repositoryMountPath  = "/repository"
+	repositoryVolumeName = "repository"
 )
 
 // Mover is the reconciliation logic for the Restic-based data mover.
@@ -76,6 +78,7 @@ type Mover struct {
 	privileged            bool
 	latestMoverStatus     *volsyncv1alpha1.MoverStatus
 	moverConfig           volsyncv1alpha1.MoverConfig
+	repositoryPVC         string
 	// Source-only fields
 	pruneInterval *int32
 	unlock        string
@@ -430,6 +433,19 @@ func (m *Mover) ensureJob(ctx context.Context, cachePVC *corev1.PersistentVolume
 		// Run mover in debug mode if required
 		envVars = utils.AppendDebugMoverEnvVar(m.owner, envVars)
 
+		volumeMounts := []corev1.VolumeMount{
+			{Name: dataVolumeName, MountPath: mountPath},
+			{Name: resticCache, MountPath: resticCacheMountPath},
+			{Name: "tempdir", MountPath: "/tmp"},
+		}
+
+		if m.repositoryPVC != "" {
+			volumeMounts = append(volumeMounts, corev1.VolumeMount{
+				Name:      repositoryVolumeName,
+				MountPath: repositoryMountPath,
+			})
+		}
+
 		podSpec.Containers = []corev1.Container{{
 			Name:    "restic",
 			Env:     envVars,
@@ -444,15 +460,12 @@ func (m *Mover) ensureJob(ctx context.Context, cachePVC *corev1.PersistentVolume
 				Privileged:             ptr.To(false),
 				ReadOnlyRootFilesystem: ptr.To(true),
 			},
-			VolumeMounts: []corev1.VolumeMount{
-				{Name: dataVolumeName, MountPath: mountPath},
-				{Name: resticCache, MountPath: resticCacheMountPath},
-				{Name: "tempdir", MountPath: "/tmp"},
-			},
+			VolumeMounts: volumeMounts,
 		}}
 		podSpec.RestartPolicy = corev1.RestartPolicyNever
 		podSpec.ServiceAccountName = sa.Name
-		podSpec.Volumes = []corev1.Volume{
+
+		volumes := []corev1.Volume{
 			{Name: dataVolumeName, VolumeSource: corev1.VolumeSource{
 				PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
 					ClaimName: dataPVC.Name,
@@ -470,6 +483,19 @@ func (m *Mover) ensureJob(ctx context.Context, cachePVC *corev1.PersistentVolume
 				}},
 			},
 		}
+
+		if m.repositoryPVC != "" {
+			volumes = append(volumes, corev1.Volume{
+				Name: repositoryVolumeName, VolumeSource: corev1.VolumeSource{
+					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+						ClaimName: m.repositoryPVC,
+						ReadOnly:  false,
+					}},
+			})
+		}
+
+		podSpec.Volumes = volumes
+
 		if m.vh.IsCopyMethodDirect() {
 			affinity, err := utils.AffinityFromVolume(ctx, m.client, logger, dataPVC)
 			if err != nil {

diff --git a/custom-scorecard-tests/config-downstream.yaml b/custom-scorecard-tests/config-downstream.yaml
@@ -135,6 +135,16 @@ stages:
     storage:
       spec:
         mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_restic_manual_pvc_copy_trigger.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_restic_manual_pvc_copy_trigger.yml
+    storage:
+      spec:
+        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_restic_with_customca_configmap.yml

diff --git a/custom-scorecard-tests/config.yaml b/custom-scorecard-tests/config.yaml
@@ -135,6 +135,16 @@ stages:
     storage:
       spec:
         mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_restic_manual_pvc_copy_trigger.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_restic_manual_pvc_copy_trigger.yml
+    storage:
+      spec:
+        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_restic_with_customca_configmap.yml

diff --git a/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml b/custom-scorecard-tests/scorecard/bases/patches/e2e-tests-stage1.yaml
@@ -121,6 +121,16 @@
     storage:
       spec:
         mountPath: {}
+  - entrypoint:
+    - volsync-custom-scorecard-tests
+    - test_restic_manual_pvc_copy_trigger.yml
+    image: quay.io/backube/volsync-custom-scorecard-tests:latest
+    labels:
+      suite: volsync-e2e
+      test: test_restic_manual_pvc_copy_trigger.yml
+    storage:
+      spec:
+        mountPath: {}
   - entrypoint:
     - volsync-custom-scorecard-tests
     - test_restic_with_customca_configmap.yml

diff --git a/docs/usage/restic/index.rst b/docs/usage/restic/index.rst
@@ -59,6 +59,10 @@ The path used in the ``RESTIC_REPOSITORY`` is the s3 bucket but can optionally
 contain a folder name within the bucket as well.  This can be useful
 if multiple PVCs are to be backed up to the same S3 bucket.
 
+When ``repositoryPVC`` is used, the ``RESTIC_REPOSITORY`` should be the path to the
+folder within the PVC where the repository is stored. The ``repositoryPVC`` will be
+mounted to ``/repository`` within the mover pod.
+
 As an example one restic-config secret could use:
 
 .. code-block:: yaml
@@ -295,6 +299,11 @@ enableFileDeletion
    A boolean indicating whether files and directories that exist on the pvc
    being restored to should be deleted if they do not exist in the restic
    snapshot being restored. The default value is ``false``.
+repositoryPVC
+   This is the name of the PVC that contains the restic repository. This is
+   useful when the repository is stored on a PVC and not in a cloud storage
+   service. The PVC referenced should exist in the same namespace where the mover
+   pods are running.
 
 Using a custom certificate authority
 ====================================

diff --git a/helm/volsync/templates/volsync.backube_replicationdestinations.yaml b/helm/volsync/templates/volsync.backube_replicationdestinations.yaml
@@ -2585,6 +2585,9 @@ spec:
                     repository:
                       description: Repository is the secret name containing repository info
                       type: string
+                    repositoryPVC:
+                      description: repositoryPVC is the name of an existing PVC containing the backup repository
+                      type: string
                     restoreAsOf:
                       description: RestoreAsOf refers to the backup that is most recent as of that time.
                       format: date-time

diff --git a/helm/volsync/templates/volsync.backube_replicationsources.yaml b/helm/volsync/templates/volsync.backube_replicationsources.yaml
@@ -2536,6 +2536,9 @@ spec:
                     repository:
                       description: Repository is the secret name containing repository info
                       type: string
+                    repositoryPVC:
+                      description: repositoryPVC is the name of an existing PVC containing the backup repository
+                      type: string
                     retain:
                       description: ResticRetainPolicy define the retain policy
                       properties: