diff --git a/api/v1/testkube.yaml b/api/v1/testkube.yaml index 50079b8db7..065638042c 100644 --- a/api/v1/testkube.yaml +++ b/api/v1/testkube.yaml @@ -8678,6 +8678,10 @@ components: type: string readinessProbe: $ref: "#/components/schemas/Probe" + pvcs: + type: object + additionalProperties: + $ref: "#/components/schemas/TestWorkflowPvcConfig" TestWorkflowServiceSpec: type: object @@ -8797,6 +8801,10 @@ components: $ref: "#/components/schemas/TestWorkflowEvent" execution: $ref: "#/components/schemas/TestWorkflowTagSchema" + pvcs: + type: object + additionalProperties: + $ref: "#/components/schemas/TestWorkflowPvcConfig" TestWorkflowTemplateSpec: type: object @@ -8835,6 +8843,10 @@ components: $ref: "#/components/schemas/TestWorkflowEvent" execution: $ref: "#/components/schemas/TestWorkflowTagSchema" + pvcs: + type: object + additionalProperties: + $ref: "#/components/schemas/TestWorkflowPvcConfig" TestWorkflowStepControl: type: object @@ -10588,6 +10600,31 @@ components: must be defined type: boolean + TestWorkflowPvcConfig: + type: object + properties: + accessModes: + description: 'Access mode for claim storage. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes' + type: array + items: + type: string + volumeMode: + description: 'Volume mode indicates the consumption of the volume as either a filesystem or block device. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#volume-mode' + $ref: "#/components/schemas/BoxedString" + resources: + description: 'Resources required for pvc' + $ref: "#/components/schemas/TestWorkflowResources" + storageClassName: + description: 'Storage class name specifies the name of a StorageClass. More info: https://kubernetes.io/docs/concepts/storage/storage-classes/' + $ref: "#/components/schemas/BoxedString" + volumeName: + description: 'Volume name is used to identify the volume' + type: string + selector: + description: Only the volumes whose labels match the selector can be bound to the claim + $ref: "#/components/schemas/LabelSelector" + # # Errors # diff --git a/cmd/tcl/testworkflow-toolkit/commands/services.go b/cmd/tcl/testworkflow-toolkit/commands/services.go index b32f3231fb..ce692cc849 100644 --- a/cmd/tcl/testworkflow-toolkit/commands/services.go +++ b/cmd/tcl/testworkflow-toolkit/commands/services.go @@ -156,6 +156,7 @@ func NewServicesCmd() *cobra.Command { Steps: []testworkflowsv1.Step{ {StepOperations: testworkflowsv1.StepOperations{Run: common.Ptr(svcSpec.StepRun)}}, }, + Pvcs: svcSpec.Pvcs, } spec.Steps[0].Run.ContainerConfig = testworkflowsv1.ContainerConfig{} diff --git a/go.mod b/go.mod index f2e215d06b..d83e255cfb 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/keygen-sh/jsonapi-go v1.2.1 github.com/keygen-sh/keygen-go/v3 v3.2.0 github.com/kubepug/kubepug v1.7.1 - github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a + github.com/kubeshop/testkube-operator v1.17.55-0.20241213144851-e0d37fb5899a github.com/minio/minio-go/v7 v7.0.66 github.com/montanaflynn/stats v0.7.1 github.com/moogar0880/problems v0.1.1 diff --git a/go.sum b/go.sum index 56f08fbd15..35bbe8851a 100644 --- a/go.sum +++ b/go.sum @@ -336,8 +336,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubepug/kubepug v1.7.1 h1:LKhfSxS8Y5mXs50v+3Lpyec+cogErDLcV7CMUuiaisw= github.com/kubepug/kubepug v1.7.1/go.mod h1:lv+HxD0oTFL7ZWjj0u6HKhMbbTIId3eG7aWIW0gyF8g= -github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a h1:xget2cwwqOL+K2Op9FPbMgfzj9lSVJAzZ9p48yxuFrE= -github.com/kubeshop/testkube-operator v1.17.55-0.20241118133003-70462ac10f4a/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk= +github.com/kubeshop/testkube-operator v1.17.55-0.20241213144851-e0d37fb5899a h1:nQDxKFu67lz9BH8isLnP7ttp7cgBWYnRDMV9VxMaYH0= +github.com/kubeshop/testkube-operator v1.17.55-0.20241213144851-e0d37fb5899a/go.mod h1:P47tw1nKQFufdsZndyq2HG2MSa0zK/lU0XpRfZtEmIk= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lithammer/fuzzysearch v1.1.8 h1:/HIuJnjHuXS8bKaiTMeeDlW2/AyIWk2brx1V8LFgLN4= diff --git a/pkg/api/v1/testkube/model_test_workflow_independent_service_spec.go b/pkg/api/v1/testkube/model_test_workflow_independent_service_spec.go index 192eeadb60..57ea947f73 100644 --- a/pkg/api/v1/testkube/model_test_workflow_independent_service_spec.go +++ b/pkg/api/v1/testkube/model_test_workflow_independent_service_spec.go @@ -41,5 +41,6 @@ type TestWorkflowIndependentServiceSpec struct { // matrix of parameters to spawn instances Matrix map[string]interface{} `json:"matrix,omitempty"` // parameters that should be distributed across sharded instances - Shards map[string]interface{} `json:"shards,omitempty"` + Shards map[string]interface{} `json:"shards,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_test_workflow_independent_step_parallel.go b/pkg/api/v1/testkube/model_test_workflow_independent_step_parallel.go index ddb97dc54c..a68339239e 100644 --- a/pkg/api/v1/testkube/model_test_workflow_independent_step_parallel.go +++ b/pkg/api/v1/testkube/model_test_workflow_independent_step_parallel.go @@ -53,4 +53,5 @@ type TestWorkflowIndependentStepParallel struct { After []TestWorkflowIndependentStep `json:"after,omitempty"` Events []TestWorkflowEvent `json:"events,omitempty"` Execution *TestWorkflowTagSchema `json:"execution,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_test_workflow_pvc_config.go b/pkg/api/v1/testkube/model_test_workflow_pvc_config.go new file mode 100644 index 0000000000..02b3f00cdb --- /dev/null +++ b/pkg/api/v1/testkube/model_test_workflow_pvc_config.go @@ -0,0 +1,21 @@ +/* + * Testkube API + * + * Testkube provides a Kubernetes-native framework for test definition, execution and results + * + * API version: 1.0.0 + * Contact: testkube@kubeshop.io + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ +package testkube + +type TestWorkflowPvcConfig struct { + // Access mode for claim storage. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes + AccessModes []string `json:"accessModes,omitempty"` + VolumeMode *BoxedString `json:"volumeMode,omitempty"` + Resources *TestWorkflowResources `json:"resources,omitempty"` + StorageClassName *BoxedString `json:"storageClassName,omitempty"` + // Volume name is used to identify the volume + VolumeName string `json:"volumeName,omitempty"` + Selector *LabelSelector `json:"selector,omitempty"` +} diff --git a/pkg/api/v1/testkube/model_test_workflow_service_spec.go b/pkg/api/v1/testkube/model_test_workflow_service_spec.go index ad915dd162..4df57b081e 100644 --- a/pkg/api/v1/testkube/model_test_workflow_service_spec.go +++ b/pkg/api/v1/testkube/model_test_workflow_service_spec.go @@ -42,5 +42,6 @@ type TestWorkflowServiceSpec struct { // matrix of parameters to spawn instances Matrix map[string]interface{} `json:"matrix,omitempty"` // parameters that should be distributed across sharded instances - Shards map[string]interface{} `json:"shards,omitempty"` + Shards map[string]interface{} `json:"shards,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_test_workflow_spec.go b/pkg/api/v1/testkube/model_test_workflow_spec.go index 87a6c0e7ee..f8a69366d4 100644 --- a/pkg/api/v1/testkube/model_test_workflow_spec.go +++ b/pkg/api/v1/testkube/model_test_workflow_spec.go @@ -23,4 +23,5 @@ type TestWorkflowSpec struct { After []TestWorkflowStep `json:"after,omitempty"` Events []TestWorkflowEvent `json:"events,omitempty"` Execution *TestWorkflowTagSchema `json:"execution,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_test_workflow_step_parallel.go b/pkg/api/v1/testkube/model_test_workflow_step_parallel.go index 7c8a9ffcca..d11b2e1d85 100644 --- a/pkg/api/v1/testkube/model_test_workflow_step_parallel.go +++ b/pkg/api/v1/testkube/model_test_workflow_step_parallel.go @@ -55,4 +55,5 @@ type TestWorkflowStepParallel struct { After []TestWorkflowStep `json:"after,omitempty"` Events []TestWorkflowEvent `json:"events,omitempty"` Execution *TestWorkflowTagSchema `json:"execution,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/api/v1/testkube/model_test_workflow_template_spec.go b/pkg/api/v1/testkube/model_test_workflow_template_spec.go index 17d16455a7..c1b9c34c40 100644 --- a/pkg/api/v1/testkube/model_test_workflow_template_spec.go +++ b/pkg/api/v1/testkube/model_test_workflow_template_spec.go @@ -22,4 +22,5 @@ type TestWorkflowTemplateSpec struct { After []TestWorkflowIndependentStep `json:"after,omitempty"` Events []TestWorkflowEvent `json:"events,omitempty"` Execution *TestWorkflowTagSchema `json:"execution,omitempty"` + Pvcs map[string]TestWorkflowPvcConfig `json:"pvcs,omitempty"` } diff --git a/pkg/mapper/testworkflows/kube_openapi.go b/pkg/mapper/testworkflows/kube_openapi.go index 9076451da8..333a2c4d12 100644 --- a/pkg/mapper/testworkflows/kube_openapi.go +++ b/pkg/mapper/testworkflows/kube_openapi.go @@ -915,6 +915,7 @@ func MapStepParallelKubeToAPI(v testworkflowsv1.StepParallel) testkube.TestWorkf Run: common.MapPtr(v.Run, MapStepRunKubeToAPI), Execute: common.MapPtr(v.Execute, MapStepExecuteKubeToAPI), Artifacts: common.MapPtr(v.Artifacts, MapStepArtifactsKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -948,6 +949,7 @@ func MapIndependentStepParallelKubeToAPI(v testworkflowsv1.IndependentStepParall Run: common.MapPtr(v.Run, MapStepRunKubeToAPI), Execute: common.MapPtr(v.Execute, MapStepExecuteKubeToAPI), Artifacts: common.MapPtr(v.Artifacts, MapStepArtifactsKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -1028,6 +1030,7 @@ func MapIndependentServiceSpecKubeToAPI(v testworkflowsv1.IndependentServiceSpec Logs: MapStringToBoxedString(v.Logs), RestartPolicy: string(v.RestartPolicy), ReadinessProbe: common.MapPtr(v.ReadinessProbe, MapProbeKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -1057,6 +1060,7 @@ func MapServiceSpecKubeToAPI(v testworkflowsv1.ServiceSpec) testkube.TestWorkflo Logs: MapStringToBoxedString(v.Logs), RestartPolicy: string(v.RestartPolicy), ReadinessProbe: common.MapPtr(v.ReadinessProbe, MapProbeKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -1134,6 +1138,7 @@ func MapSpecKubeToAPI(v testworkflowsv1.TestWorkflowSpec) testkube.TestWorkflowS After: common.MapSlice(v.After, MapStepKubeToAPI), Events: common.MapSlice(v.Events, MapEventKubeToAPI), Execution: common.MapPtr(v.Execution, MapTestWorkflowTagSchemaKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -1150,6 +1155,7 @@ func MapTemplateSpecKubeToAPI(v testworkflowsv1.TestWorkflowTemplateSpec) testku After: common.MapSlice(v.After, MapIndependentStepKubeToAPI), Events: common.MapSlice(v.Events, MapEventKubeToAPI), Execution: common.MapPtr(v.Execution, MapTestWorkflowTagSchemaKubeToAPI), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigKubeToAPI), } } @@ -1206,3 +1212,14 @@ func MapTestWorkflowTagSchemaKubeToAPI(v testworkflowsv1.TestWorkflowTagSchema) Tags: v.Tags, } } + +func MapPvcConfigKubeToAPI(v testworkflowsv1.TestWorkflowPvcConfig) testkube.TestWorkflowPvcConfig { + return testkube.TestWorkflowPvcConfig{ + AccessModes: v.AccessModes, + VolumeMode: MapStringToBoxedString(v.VolumeMode), + Resources: common.MapPtr(v.Resources, MapResourcesKubeToAPI), + StorageClassName: MapStringToBoxedString(v.StorageClassName), + VolumeName: v.VolumeName, + Selector: common.MapPtr(v.Selector, MapSelectorToAPI), + } +} diff --git a/pkg/mapper/testworkflows/openapi_kube.go b/pkg/mapper/testworkflows/openapi_kube.go index 86b9bf7ede..fa5a76abc0 100644 --- a/pkg/mapper/testworkflows/openapi_kube.go +++ b/pkg/mapper/testworkflows/openapi_kube.go @@ -951,6 +951,7 @@ func MapStepParallelAPIToKube(v testkube.TestWorkflowStepParallel) testworkflows Setup: common.MapSlice(v.Setup, MapStepAPIToKube), Steps: common.MapSlice(v.Steps, MapStepAPIToKube), After: common.MapSlice(v.After, MapStepAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), }, StepControl: testworkflowsv1.StepControl{ Paused: v.Paused, @@ -995,6 +996,7 @@ func MapIndependentStepParallelAPIToKube(v testkube.TestWorkflowIndependentStepP Setup: common.MapSlice(v.Setup, MapIndependentStepAPIToKube), Steps: common.MapSlice(v.Steps, MapIndependentStepAPIToKube), After: common.MapSlice(v.After, MapIndependentStepAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), }, StepControl: testworkflowsv1.StepControl{ Paused: v.Paused, @@ -1098,6 +1100,7 @@ func MapIndependentServiceSpecAPIToKube(v testkube.TestWorkflowIndependentServic Logs: MapBoxedStringToString(v.Logs), RestartPolicy: testworkflowsv1.ServiceRestartPolicy(v.RestartPolicy), ReadinessProbe: common.MapPtr(v.ReadinessProbe, MapProbeAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), } } @@ -1134,6 +1137,7 @@ func MapServiceSpecAPIToKube(v testkube.TestWorkflowServiceSpec) testworkflowsv1 Logs: MapBoxedStringToString(v.Logs), RestartPolicy: testworkflowsv1.ServiceRestartPolicy(v.RestartPolicy), ReadinessProbe: common.MapPtr(v.ReadinessProbe, MapProbeAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), }, } } @@ -1234,6 +1238,7 @@ func MapSpecAPIToKube(v testkube.TestWorkflowSpec) testworkflowsv1.TestWorkflowS Setup: common.MapSlice(v.Setup, MapStepAPIToKube), Steps: common.MapSlice(v.Steps, MapStepAPIToKube), After: common.MapSlice(v.After, MapStepAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), } } @@ -1253,6 +1258,7 @@ func MapTemplateSpecAPIToKube(v testkube.TestWorkflowTemplateSpec) testworkflows Setup: common.MapSlice(v.Setup, MapIndependentStepAPIToKube), Steps: common.MapSlice(v.Steps, MapIndependentStepAPIToKube), After: common.MapSlice(v.After, MapIndependentStepAPIToKube), + Pvcs: common.MapMap(v.Pvcs, MapPvcConfigAPIToKube), } } @@ -1485,3 +1491,14 @@ func MapTestWorkflowTagSchemaAPIToKube(v testkube.TestWorkflowTagSchema) testwor Tags: v.Tags, } } + +func MapPvcConfigAPIToKube(v testkube.TestWorkflowPvcConfig) testworkflowsv1.TestWorkflowPvcConfig { + return testworkflowsv1.TestWorkflowPvcConfig{ + AccessModes: v.AccessModes, + VolumeMode: MapBoxedStringToString(v.VolumeMode), + Resources: common.MapPtr(v.Resources, MapResourcesAPIToKube), + StorageClassName: MapBoxedStringToString(v.StorageClassName), + VolumeName: v.VolumeName, + Selector: common.MapPtr(v.Selector, MapLabelSelectorAPIToKube), + } +} diff --git a/pkg/testworkflows/executionworker/controller/cleanup.go b/pkg/testworkflows/executionworker/controller/cleanup.go index 1a6faa7d8b..a70d7194ee 100644 --- a/pkg/testworkflows/executionworker/controller/cleanup.go +++ b/pkg/testworkflows/executionworker/controller/cleanup.go @@ -57,6 +57,17 @@ func cleanupJobs(labelName string) func(ctx context.Context, clientSet kubernete } } +func cleanupPvcs(labelName string) func(ctx context.Context, clientSet kubernetes.Interface, namespace, id string) error { + return func(ctx context.Context, clientSet kubernetes.Interface, namespace, id string) error { + return clientSet.CoreV1().PersistentVolumeClaims(namespace).DeleteCollection(ctx, metav1.DeleteOptions{ + GracePeriodSeconds: common.Ptr(int64(0)), + PropagationPolicy: common.Ptr(metav1.DeletePropagationBackground), + }, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", labelName, id), + }) + } +} + func Cleanup(ctx context.Context, clientSet kubernetes.Interface, namespace, id string) error { var errs []error var errsMu sync.Mutex @@ -70,6 +81,8 @@ func Cleanup(ctx context.Context, clientSet kubernetes.Interface, namespace, id cleanupConfigMaps(constants.ResourceIdLabelName), cleanupSecrets(constants.RootResourceIdLabelName), cleanupSecrets(constants.ResourceIdLabelName), + cleanupPvcs(constants.RootResourceIdLabelName), + cleanupPvcs(constants.ResourceIdLabelName), } wg.Add(len(ops)) for _, op := range ops { @@ -96,6 +109,7 @@ func CleanupGroup(ctx context.Context, clientSet kubernetes.Interface, namespace cleanupPods(constants.GroupIdLabelName), cleanupConfigMaps(constants.GroupIdLabelName), cleanupSecrets(constants.GroupIdLabelName), + cleanupPvcs(constants.GroupIdLabelName), } wg.Add(len(ops)) for _, op := range ops { diff --git a/pkg/testworkflows/testworkflowconfig/expressions.go b/pkg/testworkflows/testworkflowconfig/expressions.go index 3cd5cd330b..b951bddb63 100644 --- a/pkg/testworkflows/testworkflowconfig/expressions.go +++ b/pkg/testworkflows/testworkflowconfig/expressions.go @@ -1,6 +1,12 @@ package testworkflowconfig -import "github.com/kubeshop/testkube/pkg/expressions" +import ( + "strings" + + corev1 "k8s.io/api/core/v1" + + "github.com/kubeshop/testkube/pkg/expressions" +) func CreateExecutionMachine(cfg *ExecutionConfig) expressions.Machine { return expressions.NewMachine(). @@ -77,3 +83,17 @@ func CreateWorkerMachine(cfg *WorkerConfig) expressions.Machine { }) return expressions.CombinedMachines(machine) } + +func CreatePvcMachine(pvcs []corev1.PersistentVolumeClaim) expressions.Machine { + pvcMap := make(map[string]string) + for _, pvc := range pvcs { + name := pvc.Name + if index := strings.LastIndex(name, "-"); index != -1 { + name = name[:index] + } + + pvcMap[name+".name"] = pvc.Name + } + + return expressions.NewMachine().RegisterStringMap("pvcs", pvcMap) +} diff --git a/pkg/testworkflows/testworkflowprocessor/bundle.go b/pkg/testworkflows/testworkflowprocessor/bundle.go index 84bfe8a694..fb49754391 100644 --- a/pkg/testworkflows/testworkflowprocessor/bundle.go +++ b/pkg/testworkflows/testworkflowprocessor/bundle.go @@ -27,6 +27,7 @@ type BundleOptions struct { type Bundle struct { Secrets []corev1.Secret ConfigMaps []corev1.ConfigMap + Pvcs []corev1.PersistentVolumeClaim Job batchv1.Job Signature []stage.Signature FullSignature []stage.Signature @@ -50,6 +51,9 @@ func (b *Bundle) SetGroupId(groupId string) { for i := range b.Secrets { AnnotateGroupId(&b.Secrets[i], groupId) } + for i := range b.Pvcs { + AnnotateGroupId(&b.Pvcs[i], groupId) + } } func (b *Bundle) Deploy(ctx context.Context, clientSet kubernetes.Interface, namespace string) (err error) { @@ -68,6 +72,13 @@ func (b *Bundle) Deploy(ctx context.Context, clientSet kubernetes.Interface, nam return errors.Wrap(err, "failed to deploy config maps") } } + for _, item := range b.Pvcs { + _, err = clientSet.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, &item, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "failed to deploy pvcs") + } + } + _, err = clientSet.BatchV1().Jobs(namespace).Create(ctx, &b.Job, metav1.CreateOptions{}) return errors.Wrap(err, "failed to deploy job") } diff --git a/pkg/testworkflows/testworkflowprocessor/intermediate.go b/pkg/testworkflows/testworkflowprocessor/intermediate.go index b92b268338..9be9b86fe0 100644 --- a/pkg/testworkflows/testworkflowprocessor/intermediate.go +++ b/pkg/testworkflows/testworkflowprocessor/intermediate.go @@ -4,6 +4,7 @@ import ( "fmt" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" @@ -21,9 +22,11 @@ type Intermediate interface { ConfigMaps() []corev1.ConfigMap Secrets() []corev1.Secret Volumes() []corev1.Volume + Pvcs() []corev1.PersistentVolumeClaim AppendJobConfig(cfg *testworkflowsv1.JobConfig) Intermediate AppendPodConfig(cfg *testworkflowsv1.PodConfig) Intermediate + AppendPvc(cfg map[string]corev1.PersistentVolumeClaimSpec) Intermediate AddConfigMap(configMap corev1.ConfigMap) Intermediate AddSecret(secret corev1.Secret) Intermediate @@ -47,8 +50,9 @@ type intermediate struct { Job testworkflowsv1.JobConfig `expr:"include"` // Actual Kubernetes resources to use - Secs []corev1.Secret `expr:"force"` - Cfgs []corev1.ConfigMap `expr:"force"` + Secs []corev1.Secret `expr:"force"` + Cfgs []corev1.ConfigMap `expr:"force"` + Ps []corev1.PersistentVolumeClaim `expr:"force"` // Storing files Files ConfigMapFiles `expr:"include"` @@ -87,6 +91,10 @@ func (s *intermediate) Volumes() []corev1.Volume { return append(s.Pod.Volumes, s.Files.Volumes()...) } +func (s *intermediate) Pvcs() []corev1.PersistentVolumeClaim { + return s.Ps +} + func (s *intermediate) AppendJobConfig(cfg *testworkflowsv1.JobConfig) Intermediate { s.Job = *testworkflowresolver.MergeJobConfig(&s.Job, cfg) return s @@ -97,6 +105,19 @@ func (s *intermediate) AppendPodConfig(cfg *testworkflowsv1.PodConfig) Intermedi return s } +func (s *intermediate) AppendPvc(cfg map[string]corev1.PersistentVolumeClaimSpec) Intermediate { + ref := s.NextRef() + for name, spec := range cfg { + s.Ps = append(s.Ps, corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", name, ref), + }, + Spec: spec, + }) + } + return s +} + func (s *intermediate) AddVolume(volume corev1.Volume) Intermediate { s.Pod.Volumes = append(s.Pod.Volumes, volume) return s diff --git a/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go b/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go index 119db5cbdb..a5627ed0d5 100644 --- a/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go +++ b/pkg/testworkflows/testworkflowprocessor/mock_intermediate.go @@ -150,6 +150,20 @@ func (mr *MockIntermediateMockRecorder) AppendPodConfig(arg0 interface{}) *gomoc return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendPodConfig", reflect.TypeOf((*MockIntermediate)(nil).AppendPodConfig), arg0) } +// AppendPvc mocks base method. +func (m *MockIntermediate) AppendPvc(arg0 map[string]v10.PersistentVolumeClaimSpec) Intermediate { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppendPvc", arg0) + ret0, _ := ret[0].(Intermediate) + return ret0 +} + +// AppendPvc indicates an expected call of AppendPvc. +func (mr *MockIntermediateMockRecorder) AppendPvc(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendPvc", reflect.TypeOf((*MockIntermediate)(nil).AppendPvc), arg0) +} + // ConfigMaps mocks base method. func (m *MockIntermediate) ConfigMaps() []v10.ConfigMap { m.ctrl.T.Helper() @@ -220,6 +234,20 @@ func (mr *MockIntermediateMockRecorder) PodConfig() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PodConfig", reflect.TypeOf((*MockIntermediate)(nil).PodConfig)) } +// Pvcs mocks base method. +func (m *MockIntermediate) Pvcs() []v10.PersistentVolumeClaim { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Pvcs") + ret0, _ := ret[0].([]v10.PersistentVolumeClaim) + return ret0 +} + +// Pvcs indicates an expected call of Pvcs. +func (mr *MockIntermediateMockRecorder) Pvcs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pvcs", reflect.TypeOf((*MockIntermediate)(nil).Pvcs)) +} + // Secrets mocks base method. func (m *MockIntermediate) Secrets() []v10.Secret { m.ctrl.T.Helper() diff --git a/pkg/testworkflows/testworkflowprocessor/processor.go b/pkg/testworkflows/testworkflowprocessor/processor.go index 1a0d460a69..43e8b054a4 100644 --- a/pkg/testworkflows/testworkflowprocessor/processor.go +++ b/pkg/testworkflows/testworkflowprocessor/processor.go @@ -23,6 +23,7 @@ import ( "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/action/actiontypes/lite" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/constants" "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor/stage" + "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowresolver" ) //go:generate mockgen -destination=./mock_processor.go -package=testworkflowprocessor "github.com/kubeshop/testkube/pkg/testworkflows/testworkflowprocessor" Processor @@ -106,7 +107,8 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo // Initialize intermediate layer layer := NewIntermediate(). AppendPodConfig(workflow.Spec.Pod). - AppendJobConfig(workflow.Spec.Job) + AppendJobConfig(workflow.Spec.Job). + AppendPvc(common.MapMap(workflow.Spec.Pvcs, testworkflowresolver.ConvertTestWorkflowPvcConfigToPersistentVolumeClaimSpec)) layer.ContainerDefaults(). ApplyCR(constants.DefaultContainerConfig.DeepCopy()). AppendVolumeMounts(layer.AddEmptyDirVolume(nil, constants.DefaultInternalPath)). @@ -117,7 +119,8 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo machines = append(machines, createSecretMachine(mapEnv), testworkflowconfig.CreateWorkerMachine(&options.Config.Worker), - testworkflowconfig.CreateResourceMachine(&options.Config.Resource)) + testworkflowconfig.CreateResourceMachine(&options.Config.Resource), + testworkflowconfig.CreatePvcMachine(layer.Pvcs())) // Fetch resource root and resource ID if options.Config.Resource.Id == "" { @@ -162,6 +165,16 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo } } + // Finalize Pvcs + pvcs := layer.Pvcs() + for i := range pvcs { + AnnotateControlledBy(&pvcs[i], options.Config.Resource.RootId, options.Config.Resource.Id) + err = expressions.FinalizeForce(&pvcs[i], machines...) + if err != nil { + return nil, errors.Wrap(err, "finalizing Pvc") + } + } + // Finalize Secrets secrets := append(layer.Secrets(), options.Secrets...) for i := range secrets { @@ -420,6 +433,7 @@ func (p *processor) Bundle(ctx context.Context, workflow *testworkflowsv1.TestWo bundle = &Bundle{ ConfigMaps: configMaps, Secrets: secrets, + Pvcs: pvcs, Job: jobSpec, Signature: sig, FullSignature: fullSig, diff --git a/pkg/testworkflows/testworkflowresolver/apply.go b/pkg/testworkflows/testworkflowresolver/apply.go index a3afdc9ea8..07ee151165 100644 --- a/pkg/testworkflows/testworkflowresolver/apply.go +++ b/pkg/testworkflows/testworkflowresolver/apply.go @@ -63,6 +63,7 @@ func injectTemplateToSpec(spec *testworkflowsv1.TestWorkflowSpec, template testw spec.Job = MergeJobConfig(template.Spec.Job, spec.Job) spec.Events = append(template.Spec.Events, spec.Events...) spec.Execution = MergeExecution(template.Spec.Execution, spec.Execution) + spec.Pvcs = MergeMap(template.Spec.Pvcs, spec.Pvcs) // Apply basic configuration spec.Content = MergeContent(template.Spec.Content, spec.Content) @@ -125,6 +126,7 @@ func InjectServiceTemplate(svc *testworkflowsv1.ServiceSpec, template testworkfl svc.Pod = MergePodConfig(template.Spec.Pod, svc.Pod) svc.Content = MergeContent(template.Spec.Content, svc.Content) svc.ContainerConfig = *MergeContainerConfig(template.Spec.Container, &svc.ContainerConfig) + svc.Pvcs = MergeMap(template.Spec.Pvcs, svc.Pvcs) return nil } diff --git a/pkg/testworkflows/testworkflowresolver/merge.go b/pkg/testworkflows/testworkflowresolver/merge.go index 1d503f1b84..3ebf623ab0 100644 --- a/pkg/testworkflows/testworkflowresolver/merge.go +++ b/pkg/testworkflows/testworkflowresolver/merge.go @@ -12,6 +12,8 @@ import ( "maps" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" testworkflowsv1 "github.com/kubeshop/testkube-operator/api/testworkflows/v1" "github.com/kubeshop/testkube/internal/common" @@ -373,6 +375,7 @@ func ConvertIndependentStepParallelToStepParallel(step testworkflowsv1.Independe Setup: common.MapSlice(step.TestWorkflowTemplateSpec.Setup, ConvertIndependentStepToStep), Steps: common.MapSlice(step.TestWorkflowTemplateSpec.Steps, ConvertIndependentStepToStep), After: common.MapSlice(step.TestWorkflowTemplateSpec.After, ConvertIndependentStepToStep), + Pvcs: step.TestWorkflowTemplateSpec.Pvcs, }, StepControl: step.StepControl, StepOperations: step.StepOperations, @@ -414,3 +417,60 @@ func MergeTags(dst, src map[string]string) map[string]string { return dst } + +func ConvertTestWorkflowPvcConfigToPersistentVolumeClaimSpec(pvc testworkflowsv1.TestWorkflowPvcConfig) corev1.PersistentVolumeClaimSpec { + return corev1.PersistentVolumeClaimSpec{ + AccessModes: common.MapSlice(pvc.AccessModes, func(s string) corev1.PersistentVolumeAccessMode { + return corev1.PersistentVolumeAccessMode(s) + }), + Selector: pvc.Selector, + Resources: ConvertResourcesToVolumeResourceRequirements(pvc.Resources), + VolumeName: pvc.VolumeName, + StorageClassName: pvc.StorageClassName, + VolumeMode: (*corev1.PersistentVolumeMode)(pvc.VolumeMode), + } +} + +func ConvertResourcesToVolumeResourceRequirements(r *testworkflowsv1.Resources) corev1.VolumeResourceRequirements { + var limits, requests corev1.ResourceList + if r != nil { + if len(r.Limits) != 0 { + limits = make(corev1.ResourceList) + } + + if len(r.Requests) != 0 { + requests = make(corev1.ResourceList) + } + + for key, value := range r.Limits { + var quantity resource.Quantity + if value.Type == intstr.Int { + quantity = *resource.NewQuantity(int64(value.IntVal), resource.BinarySI) + } + + if value.Type == intstr.String { + quantity, _ = resource.ParseQuantity(value.String()) + } + + limits[key] = quantity + } + + for key, value := range r.Requests { + var quantity resource.Quantity + if value.Type == intstr.Int { + quantity = *resource.NewQuantity(int64(value.IntVal), resource.BinarySI) + } + + if value.Type == intstr.String { + quantity, _ = resource.ParseQuantity(value.String()) + } + + requests[key] = quantity + } + } + + return corev1.VolumeResourceRequirements{ + Limits: limits, + Requests: requests, + } +}