feat: [TKC-2895] pvc config

api/v1/testkube.yaml

@@ -8678,6 +8678,10 @@ components:
             type: string
           readinessProbe:
             $ref: "#/components/schemas/Probe"
+          pvcs:
+            type: object
+            additionalProperties:
+              $ref: "#/components/schemas/TestWorkflowPvcConfig"
 
     TestWorkflowServiceSpec:
       type: object
@@ -8797,6 +8801,10 @@ components:
             $ref: "#/components/schemas/TestWorkflowEvent"
         execution:
           $ref: "#/components/schemas/TestWorkflowTagSchema"
+        pvcs:
+          type: object
+          additionalProperties:
+            $ref: "#/components/schemas/TestWorkflowPvcConfig"
 
     TestWorkflowTemplateSpec:
       type: object
@@ -8835,6 +8843,10 @@ components:
             $ref: "#/components/schemas/TestWorkflowEvent"
         execution:
           $ref: "#/components/schemas/TestWorkflowTagSchema"
+        pvcs:
+          type: object
+          additionalProperties:
+            $ref: "#/components/schemas/TestWorkflowPvcConfig"
 
     TestWorkflowStepControl:
       type: object
@@ -10588,6 +10600,31 @@ components:
                 must be defined
               type: boolean
 
+    TestWorkflowPvcConfig:
+      type: object
+      properties:
+        accessModes:
+          description: 'Access mode for claim storage. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes'
+          type: array
+          items:
+            type: string
+        volumeMode:
+          description: 'Volume mode indicates the consumption of the volume as either a filesystem or block device.
+            More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#volume-mode'
+          $ref: "#/components/schemas/BoxedString"
+        resources:
+          description: 'Resources required for pvc'
+          $ref: "#/components/schemas/TestWorkflowResources"
+        storageClassName:
+          description: 'Storage class name specifies the name of a StorageClass. More info: https://kubernetes.io/docs/concepts/storage/storage-classes/'
+          $ref: "#/components/schemas/BoxedString"
+        volumeName:
+          description: 'Volume name is used to identify the volume'
+          type: string
+        selector:
+          description: Only the volumes whose labels match the selector can be bound to the claim
+          $ref: "#/components/schemas/LabelSelector"
+
     #
     # Errors
     #

cmd/tcl/testworkflow-toolkit/commands/services.go

@@ -156,6 +156,7 @@ func NewServicesCmd() *cobra.Command {
 						Steps: []testworkflowsv1.Step{
 							{StepOperations: testworkflowsv1.StepOperations{Run: common.Ptr(svcSpec.StepRun)}},
 						},
+						Pvcs: svcSpec.Pvcs,
 					}
 					spec.Steps[0].Run.ContainerConfig = testworkflowsv1.ContainerConfig{}
 

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/keygen-sh/jsonapi-go v1.2.1
 	github.com/keygen-sh/keygen-go/v3 v3.2.0
 	github.com/kubepug/kubepug v1.7.1
-	github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a
+	github.com/kubeshop/testkube-operator v1.17.55-0.20241213144851-e0d37fb5899a
 	github.com/minio/minio-go/v7 v7.0.66
 	github.com/montanaflynn/stats v0.7.1
 	github.com/moogar0880/problems v0.1.1

go.sum

@@ -336,8 +336,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kubepug/kubepug v1.7.1 h1:LKhfSxS8Y5mXs50v+3Lpyec+cogErDLcV7CMUuiaisw=
 github.com/kubepug/kubepug v1.7.1/go.mod h1:lv+HxD0oTFL7ZWjj0u6HKhMbbTIId3eG7aWIW0gyF8g=
-github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a h1:xget2cwwqOL+K2Op9FPbMgfzj9lSVJAzZ9p48yxuFrE=
-github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk=
+github.com/kubeshop/testkube-operator v1.17.55-0.20241213144851-e0d37fb5899a h1:nQDxKFu67lz9BH8isLnP7ttp7cgBWYnRDMV9VxMaYH0=
+github.com/kubeshop/testkube-operator v1.17.55-0.20241213144851-e0d37fb5899a/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4=

pkg/api/v1/testkube/model_test_workflow_independent_service_spec.go

@@ -41,5 +41,6 @@ type TestWorkflowIndependentServiceSpec struct {
 	// matrix of parameters to spawn instances
 	Matrix map[string]interface{} `json:"matrix,omitempty"`
 	// parameters that should be distributed across sharded instances
-	Shards map[string]interface{} `json:"shards,omitempty"`
+	Shards map[string]interface{}           `json:"shards,omitempty"`
+	Pvcs   map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"`
 }

pkg/api/v1/testkube/model_test_workflow_independent_step_parallel.go

@@ -53,4 +53,5 @@ type TestWorkflowIndependentStepParallel struct {
 	After     []TestWorkflowIndependentStep                 `json:"after,omitempty"`
 	Events    []TestWorkflowEvent                           `json:"events,omitempty"`
 	Execution *TestWorkflowTagSchema                        `json:"execution,omitempty"`
+	Pvcs      map[string]TestWorkflowPvcConfig              `json:"pvcs,omitempty"`
 }

pkg/api/v1/testkube/model_test_workflow_pvc_config.go

@@ -0,0 +1,21 @@
+/*
+ * Testkube API
+ *
+ * Testkube provides a Kubernetes-native framework for test definition, execution and results
+ *
+ * API version: 1.0.0
+ * Contact: [email protected]
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+package testkube
+
+type TestWorkflowPvcConfig struct {
+	// Access mode for claim storage. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes
+	AccessModes      []string               `json:"accessModes,omitempty"`
+	VolumeMode       *BoxedString           `json:"volumeMode,omitempty"`
+	Resources        *TestWorkflowResources `json:"resources,omitempty"`
+	StorageClassName *BoxedString           `json:"storageClassName,omitempty"`
+	// Volume name is used to identify the volume
+	VolumeName string         `json:"volumeName,omitempty"`
+	Selector   *LabelSelector `json:"selector,omitempty"`
+}

pkg/api/v1/testkube/model_test_workflow_service_spec.go

@@ -42,5 +42,6 @@ type TestWorkflowServiceSpec struct {
 	// matrix of parameters to spawn instances
 	Matrix map[string]interface{} `json:"matrix,omitempty"`
 	// parameters that should be distributed across sharded instances
-	Shards map[string]interface{} `json:"shards,omitempty"`
+	Shards map[string]interface{}           `json:"shards,omitempty"`
+	Pvcs   map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"`
 }

pkg/api/v1/testkube/model_test_workflow_spec.go

@@ -23,4 +23,5 @@ type TestWorkflowSpec struct {
 	After     []TestWorkflowStep                     `json:"after,omitempty"`
 	Events    []TestWorkflowEvent                    `json:"events,omitempty"`
 	Execution *TestWorkflowTagSchema                 `json:"execution,omitempty"`
+	Pvcs      map[string]TestWorkflowPvcConfig       `json:"pvcs,omitempty"`
 }

pkg/api/v1/testkube/model_test_workflow_step_parallel.go

@@ -55,4 +55,5 @@ type TestWorkflowStepParallel struct {
 	After     []TestWorkflowStep                     `json:"after,omitempty"`
 	Events    []TestWorkflowEvent                    `json:"events,omitempty"`
 	Execution *TestWorkflowTagSchema                 `json:"execution,omitempty"`
+	Pvcs      map[string]TestWorkflowPvcConfig       `json:"pvcs,omitempty"`
 }

pkg/api/v1/testkube/model_test_workflow_template_spec.go

@@ -22,4 +22,5 @@ type TestWorkflowTemplateSpec struct {
 	After     []TestWorkflowIndependentStep                 `json:"after,omitempty"`
 	Events    []TestWorkflowEvent                           `json:"events,omitempty"`
 	Execution *TestWorkflowTagSchema                        `json:"execution,omitempty"`
+	Pvcs      map[string]TestWorkflowPvcConfig              `json:"pvcs,omitempty"`
 }

pkg/mapper/testworkflows/kube_openapi.go

@@ -915,6 +915,7 @@ func MapStepParallelKubeToAPI(v testworkflowsv1.StepParallel) testkube.TestWorkf
 		Run:         common.MapPtr(v.Run, MapStepRunKubeToAPI),
 		Execute:     common.MapPtr(v.Execute, MapStepExecuteKubeToAPI),
 		Artifacts:   common.MapPtr(v.Artifacts, MapStepArtifactsKubeToAPI),
+		Pvcs:        common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI),
 	}
 }
 
@@ -948,6 +949,7 @@ func MapIndependentStepParallelKubeToAPI(v testworkflowsv1.IndependentStepParall
 		Run:         common.MapPtr(v.Run, MapStepRunKubeToAPI),
 		Execute:     common.MapPtr(v.Execute, MapStepExecuteKubeToAPI),
 		Artifacts:   common.MapPtr(v.Artifacts, MapStepArtifactsKubeToAPI),
+		Pvcs:        common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI),
 	}
 }
 
@@ -1028,6 +1030,7 @@ func MapIndependentServiceSpecKubeToAPI(v testworkflowsv1.IndependentServiceSpec
 		Logs:            MapStringToBoxedString(v.Logs),
 		RestartPolicy:   string(v.RestartPolicy),
 		ReadinessProbe:  common.MapPtr(v.ReadinessProbe, MapProbeKubeToAPI),
+		Pvcs:            common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI),
 	}
 }
 
@@ -1057,6 +1060,7 @@ func MapServiceSpecKubeToAPI(v testworkflowsv1.ServiceSpec) testkube.TestWorkflo
 		Logs:            MapStringToBoxedString(v.Logs),
 		RestartPolicy:   string(v.RestartPolicy),
 		ReadinessProbe:  common.MapPtr(v.ReadinessProbe, MapProbeKubeToAPI),
+		Pvcs:            common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI),
 	}
 }
 
@@ -1134,6 +1138,7 @@ func MapSpecKubeToAPI(v testworkflowsv1.TestWorkflowSpec) testkube.TestWorkflowS
 		After:     common.MapSlice(v.After, MapStepKubeToAPI),
 		Events:    common.MapSlice(v.Events, MapEventKubeToAPI),
 		Execution: common.MapPtr(v.Execution, MapTestWorkflowTagSchemaKubeToAPI),
+		Pvcs:      common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI),
 	}
 }
 
@@ -1150,6 +1155,7 @@ func MapTemplateSpecKubeToAPI(v testworkflowsv1.TestWorkflowTemplateSpec) testku
 		After:     common.MapSlice(v.After, MapIndependentStepKubeToAPI),
 		Events:    common.MapSlice(v.Events, MapEventKubeToAPI),
 		Execution: common.MapPtr(v.Execution, MapTestWorkflowTagSchemaKubeToAPI),
+		Pvcs:      common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI),
 	}
 }
 
@@ -1206,3 +1212,14 @@ func MapTestWorkflowTagSchemaKubeToAPI(v testworkflowsv1.TestWorkflowTagSchema)
 		Tags: v.Tags,
 	}
 }
+
+func MapPvcConfigKubeToAPI(v testworkflowsv1.TestWorkflowPvcConfig) testkube.TestWorkflowPvcConfig {
+	return testkube.TestWorkflowPvcConfig{
+		AccessModes:      v.AccessModes,
+		VolumeMode:       MapStringToBoxedString(v.VolumeMode),
+		Resources:        common.MapPtr(v.Resources, MapResourcesKubeToAPI),
+		StorageClassName: MapStringToBoxedString(v.StorageClassName),
+		VolumeName:       v.VolumeName,
+		Selector:         common.MapPtr(v.Selector, MapSelectorToAPI),
+	}
+}

pkg/mapper/testworkflows/openapi_kube.go

@@ -951,6 +951,7 @@ func MapStepParallelAPIToKube(v testkube.TestWorkflowStepParallel) testworkflows
 			Setup: common.MapSlice(v.Setup, MapStepAPIToKube),
 			Steps: common.MapSlice(v.Steps, MapStepAPIToKube),
 			After: common.MapSlice(v.After, MapStepAPIToKube),
+			Pvcs:  common.MapMap(v.Pvcs, MapPvcConfigAPIToKube),
 		},
 		StepControl: testworkflowsv1.StepControl{
 			Paused:   v.Paused,
@@ -995,6 +996,7 @@ func MapIndependentStepParallelAPIToKube(v testkube.TestWorkflowIndependentStepP
 			Setup: common.MapSlice(v.Setup, MapIndependentStepAPIToKube),
 			Steps: common.MapSlice(v.Steps, MapIndependentStepAPIToKube),
 			After: common.MapSlice(v.After, MapIndependentStepAPIToKube),
+			Pvcs:  common.MapMap(v.Pvcs, MapPvcConfigAPIToKube),
 		},
 		StepControl: testworkflowsv1.StepControl{
 			Paused:   v.Paused,
@@ -1098,6 +1100,7 @@ func MapIndependentServiceSpecAPIToKube(v testkube.TestWorkflowIndependentServic
 		Logs:           MapBoxedStringToString(v.Logs),
 		RestartPolicy:  testworkflowsv1.ServiceRestartPolicy(v.RestartPolicy),
 		ReadinessProbe: common.MapPtr(v.ReadinessProbe, MapProbeAPIToKube),
+		Pvcs:           common.MapMap(v.Pvcs, MapPvcConfigAPIToKube),
 	}
 }
 
@@ -1134,6 +1137,7 @@ func MapServiceSpecAPIToKube(v testkube.TestWorkflowServiceSpec) testworkflowsv1
 			Logs:           MapBoxedStringToString(v.Logs),
 			RestartPolicy:  testworkflowsv1.ServiceRestartPolicy(v.RestartPolicy),
 			ReadinessProbe: common.MapPtr(v.ReadinessProbe, MapProbeAPIToKube),
+			Pvcs:           common.MapMap(v.Pvcs, MapPvcConfigAPIToKube),
 		},
 	}
 }
@@ -1234,6 +1238,7 @@ func MapSpecAPIToKube(v testkube.TestWorkflowSpec) testworkflowsv1.TestWorkflowS
 		Setup:    common.MapSlice(v.Setup, MapStepAPIToKube),
 		Steps:    common.MapSlice(v.Steps, MapStepAPIToKube),
 		After:    common.MapSlice(v.After, MapStepAPIToKube),
+		Pvcs:     common.MapMap(v.Pvcs, MapPvcConfigAPIToKube),
 	}
 }
 
@@ -1253,6 +1258,7 @@ func MapTemplateSpecAPIToKube(v testkube.TestWorkflowTemplateSpec) testworkflows
 		Setup:    common.MapSlice(v.Setup, MapIndependentStepAPIToKube),
 		Steps:    common.MapSlice(v.Steps, MapIndependentStepAPIToKube),
 		After:    common.MapSlice(v.After, MapIndependentStepAPIToKube),
+		Pvcs:     common.MapMap(v.Pvcs, MapPvcConfigAPIToKube),
 	}
 }
 
@@ -1485,3 +1491,14 @@ func MapTestWorkflowTagSchemaAPIToKube(v testkube.TestWorkflowTagSchema) testwor
 		Tags: v.Tags,
 	}
 }
+
+func MapPvcConfigAPIToKube(v testkube.TestWorkflowPvcConfig) testworkflowsv1.TestWorkflowPvcConfig {
+	return testworkflowsv1.TestWorkflowPvcConfig{
+		AccessModes:      v.AccessModes,
+		VolumeMode:       MapBoxedStringToString(v.VolumeMode),
+		Resources:        common.MapPtr(v.Resources, MapResourcesAPIToKube),
+		StorageClassName: MapBoxedStringToString(v.StorageClassName),
+		VolumeName:       v.VolumeName,
+		Selector:         common.MapPtr(v.Selector, MapLabelSelectorAPIToKube),
+	}
+}

pkg/testworkflows/executionworker/controller/cleanup.go

@@ -57,6 +57,17 @@ func cleanupJobs(labelName string) func(ctx context.Context, clientSet kubernete
 	}
 }
 
+func cleanupPvcs(labelName string) func(ctx context.Context, clientSet kubernetes.Interface, namespace, id string) error {
+	return func(ctx context.Context, clientSet kubernetes.Interface, namespace, id string) error {
+		return clientSet.CoreV1().PersistentVolumeClaims(namespace).DeleteCollection(ctx, metav1.DeleteOptions{
+			GracePeriodSeconds: common.Ptr(int64(0)),
+			PropagationPolicy:  common.Ptr(metav1.DeletePropagationBackground),
+		}, metav1.ListOptions{
+			LabelSelector: fmt.Sprintf("%s=%s", labelName, id),
+		})
+	}
+}
+
 func Cleanup(ctx context.Context, clientSet kubernetes.Interface, namespace, id string) error {
 	var errs []error
 	var errsMu sync.Mutex
@@ -70,6 +81,8 @@ func Cleanup(ctx context.Context, clientSet kubernetes.Interface, namespace, id
 		cleanupConfigMaps(constants.ResourceIdLabelName),
 		cleanupSecrets(constants.RootResourceIdLabelName),
 		cleanupSecrets(constants.ResourceIdLabelName),
+		cleanupPvcs(constants.RootResourceIdLabelName),
+		cleanupPvcs(constants.ResourceIdLabelName),
 	}
 	wg.Add(len(ops))
 	for _, op := range ops {
@@ -96,6 +109,7 @@ func CleanupGroup(ctx context.Context, clientSet kubernetes.Interface, namespace
 		cleanupPods(constants.GroupIdLabelName),
 		cleanupConfigMaps(constants.GroupIdLabelName),
 		cleanupSecrets(constants.GroupIdLabelName),
+		cleanupPvcs(constants.GroupIdLabelName),
 	}
 	wg.Add(len(ops))
 	for _, op := range ops {

pkg/testworkflows/testworkflowconfig/expressions.go

@@ -1,6 +1,12 @@
 package testworkflowconfig
 
-import "github.com/kubeshop/testkube/pkg/expressions"
+import (
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/kubeshop/testkube/pkg/expressions"
+)
 
 func CreateExecutionMachine(cfg *ExecutionConfig) expressions.Machine {
 	return expressions.NewMachine().
@@ -77,3 +83,17 @@ func CreateWorkerMachine(cfg *WorkerConfig) expressions.Machine {
 		})
 	return expressions.CombinedMachines(machine)
 }
+
+func CreatePvcMachine(pvcs []corev1.PersistentVolumeClaim) expressions.Machine {
+	pvcMap := make(map[string]string)
+	for _, pvc := range pvcs {
+		name := pvc.Name
+		if index := strings.LastIndex(name, "-"); index != -1 {
+			name = name[:index]
+		}
+
+		pvcMap[name+".name"] = pvc.Name
+	}
+
+	return expressions.NewMachine().RegisterStringMap("pvcs", pvcMap)
+}

pkg/testworkflows/testworkflowprocessor/bundle.go

@@ -27,6 +27,7 @@ type BundleOptions struct {
 type Bundle struct {
 	Secrets       []corev1.Secret
 	ConfigMaps    []corev1.ConfigMap
+	Pvcs          []corev1.PersistentVolumeClaim
 	Job           batchv1.Job
 	Signature     []stage.Signature
 	FullSignature []stage.Signature
@@ -50,6 +51,9 @@ func (b *Bundle) SetGroupId(groupId string) {
 	for i := range b.Secrets {
 		AnnotateGroupId(&b.Secrets[i], groupId)
 	}
+	for i := range b.Pvcs {
+		AnnotateGroupId(&b.Pvcs[i], groupId)
+	}
 }
 
 func (b *Bundle) Deploy(ctx context.Context, clientSet kubernetes.Interface, namespace string) (err error) {
@@ -68,6 +72,13 @@ func (b *Bundle) Deploy(ctx context.Context, clientSet kubernetes.Interface, nam
 			return errors.Wrap(err, "failed to deploy config maps")
 		}
 	}
+	for _, item := range b.Pvcs {
+		_, err = clientSet.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, &item, metav1.CreateOptions{})
+		if err != nil {
+			return errors.Wrap(err, "failed to deploy pvcs")
+		}
+	}
+
 	_, err = clientSet.BatchV1().Jobs(namespace).Create(ctx, &b.Job, metav1.CreateOptions{})
 	return errors.Wrap(err, "failed to deploy job")
 }