Tenant overrides for provisioning manifests (#55)

* Tenant overrides for provisioning manifests

* Swiched to interface

* Refactoring

* Deleted unused file

go.mod

@@ -35,6 +35,7 @@ require (
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect

go.sum

@@ -1,10 +1,14 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
+dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM=
 github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I=
 github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
+github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
 github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
 github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da h1:KjTM2ks9d14ZYCvmHS9iAKVt9AyzRSqNU1qabPih5BY=
@@ -122,6 +126,7 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=

helm/crds/provisioning.totalsoft.ro_azuredatabases.yaml

@@ -98,6 +98,13 @@ spec:
                 - resourceGroupName
                 - serverName
                 type: object
+              tenantOverrides:
+                additionalProperties:
+                  x-kubernetes-preserve-unknown-fields: true
+                description: Overrides for tenants. Dictionary with tenant name as
+                  key, spec override as value. The spec override has the same structure
+                  as Spec
+                type: object
             required:
             - dbName
             - domainRef

helm/crds/provisioning.totalsoft.ro_azuremanageddatabases.yaml

@@ -116,6 +116,13 @@ spec:
                 - backupFileName
                 - storageContainer
                 type: object
+              tenantOverrides:
+                additionalProperties:
+                  x-kubernetes-preserve-unknown-fields: true
+                description: Overrides for tenants. Dictionary with tenant name as
+                  key, spec override as value. The spec override has the same structure
+                  as Spec
+                type: object
             required:
             - dbName
             - domainRef

helm/crds/provisioning.totalsoft.ro_azurevirtualdesktops.yaml

@@ -175,6 +175,13 @@ spec:
               subnetId:
                 description: 'Subnet of the VNet used by the virtual machine eg: /subscriptions/05a50a12-6628-4627-bd30-19932dac39f8/resourceGroups/charismaonline.qa/providers/Microsoft.Network/virtualNetworks/charismaonline-vnet/subnets/default'
                 type: string
+              tenantOverrides:
+                additionalProperties:
+                  x-kubernetes-preserve-unknown-fields: true
+                description: Overrides for tenants. Dictionary with tenant name as
+                  key, spec override as value. The spec override has the same structure
+                  as Spec
+                type: object
               users:
                 properties:
                   admins:

helm/crds/provisioning.totalsoft.ro_azurevirtualmachines.yaml

@@ -160,6 +160,13 @@ spec:
               subnetId:
                 description: 'Subnet of the VNet used by the virtual machine eg: /subscriptions/05a50a12-6628-4627-bd30-19932dac39f8/resourceGroups/charismaonline.qa/providers/Microsoft.Network/virtualNetworks/charismaonline-vnet/subnets/default'
                 type: string
+              tenantOverrides:
+                additionalProperties:
+                  x-kubernetes-preserve-unknown-fields: true
+                description: Overrides for tenants. Dictionary with tenant name as
+                  key, spec override as value. The spec override has the same structure
+                  as Spec
+                type: object
               vmName:
                 description: Virtual Machine name prefix. Will have platform and tenant
                   suffix.

helm/crds/provisioning.totalsoft.ro_helmreleases.yaml

@@ -827,6 +827,13 @@ spec:
                 - chart
                 - interval
                 type: object
+              tenantOverrides:
+                additionalProperties:
+                  x-kubernetes-preserve-unknown-fields: true
+                description: Overrides for tenants. Dictionary with tenant name as
+                  key, spec override as value. The spec override has the same structure
+                  as Spec
+                type: object
             required:
             - domainRef
             - platformRef

internal/controllers/provisioning/provisioners/pulumi/azure_virtual_desktop_test.go

@@ -56,7 +56,9 @@ func newVirtualDesktop(name, platform string) *provisioningv1.AzureVirtualDeskto
 			Namespace: metav1.NamespaceDefault,
 		},
 		Spec: provisioningv1.AzureVirtualDesktopSpec{
-			PlatformRef:         platform,
+			ProvisioningMeta: provisioningv1.ProvisioningMeta{
+				PlatformRef: platform,
+			},
 			HostPoolName:        "test-vm",
 			VmSize:              "Standard_B1s",
 			OSDiskType:          "Standard_LRS",

internal/controllers/provisioning/provisioners/pulumi/azure_virtual_machine_test.go

@@ -56,7 +56,9 @@ func newVm(name, platform string) *provisioningv1.AzureVirtualMachine {
 			Namespace: metav1.NamespaceDefault,
 		},
 		Spec: provisioningv1.AzureVirtualMachineSpec{
-			PlatformRef:            platform,
+			ProvisioningMeta: provisioningv1.ProvisioningMeta{
+				PlatformRef: platform,
+			},
 			VmName:                 "test-vm",
 			VmSize:                 "Standard_B1s",
 			OSDiskType:             "Standard_LRS",

internal/controllers/provisioning/provisioners/pulumi/helm_release_test.go

@@ -108,7 +108,9 @@ func newHr(name, platform string) *provisioningv1.HelmRelease {
 			Namespace: metav1.NamespaceDefault,
 		},
 		Spec: provisioningv1.HelmReleaseSpec{
-			PlatformRef: platform,
+			ProvisioningMeta: provisioningv1.ProvisioningMeta{
+				PlatformRef: platform,
+			},
 			Release: flux.HelmReleaseSpec{
 				Interval: metav1.Duration{Duration: 10 * time.Second},
 				Chart: flux.HelmChartTemplate{

internal/controllers/provisioning/provisioning_controller.go

@@ -2,6 +2,7 @@ package provisioning
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"reflect"
 	"strings"
@@ -10,6 +11,7 @@ import (
 	"k8s.io/utils/strings/slices"
 
 	corev1 "k8s.io/api/core/v1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -31,6 +33,8 @@ import (
 	informers "totalsoft.ro/platform-controllers/pkg/generated/informers/externalversions"
 	platformInformersv1 "totalsoft.ro/platform-controllers/pkg/generated/informers/externalversions/platform/v1alpha1"
 	provisioningInformersv1 "totalsoft.ro/platform-controllers/pkg/generated/informers/externalversions/provisioning/v1alpha1"
+
+	"dario.cat/mergo"
 )
 
 const (
@@ -253,6 +257,11 @@ func (c *ProvisioningController) syncHandler(key string) error {
 	n := 0
 	for _, db := range azureDbs {
 		if db.Spec.PlatformRef == platform && slices.Contains(tenant.Spec.DomainRefs, db.Spec.DomainRef) {
+			err := applyTenantOverrides(db, tenant.Name)
+			if err != nil {
+				return err
+			}
+
 			azureDbs[n] = db
 			n++
 		}
@@ -267,6 +276,11 @@ func (c *ProvisioningController) syncHandler(key string) error {
 	n = 0
 	for _, db := range azureManagedDbs {
 		if db.Spec.PlatformRef == platform && slices.Contains(tenant.Spec.DomainRefs, db.Spec.DomainRef) {
+			err := applyTenantOverrides(db, tenant.Name)
+			if err != nil {
+				return err
+			}
+
 			azureManagedDbs[n] = db
 			n++
 		}
@@ -281,6 +295,11 @@ func (c *ProvisioningController) syncHandler(key string) error {
 	n = 0
 	for _, hr := range helmReleases {
 		if hr.Spec.PlatformRef == platform && slices.Contains(tenant.Spec.DomainRefs, hr.Spec.DomainRef) {
+			err := applyTenantOverrides(hr, tenant.Name)
+			if err != nil {
+				return err
+			}
+
 			helmReleases[n] = hr
 			n++
 		}
@@ -295,6 +314,11 @@ func (c *ProvisioningController) syncHandler(key string) error {
 	n = 0
 	for _, vm := range azureVirtualMachines {
 		if vm.Spec.PlatformRef == platform && slices.Contains(tenant.Spec.DomainRefs, vm.Spec.DomainRef) {
+			err := applyTenantOverrides(vm, tenant.Name)
+			if err != nil {
+				return err
+			}
+
 			azureVirtualMachines[n] = vm
 			n++
 		}
@@ -307,9 +331,14 @@ func (c *ProvisioningController) syncHandler(key string) error {
 	}
 
 	n = 0
-	for _, vm := range azureVirtualDesktops {
-		if vm.Spec.PlatformRef == platform && slices.Contains(tenant.Spec.DomainRefs, vm.Spec.DomainRef) {
-			azureVirtualDesktops[n] = vm
+	for _, avd := range azureVirtualDesktops {
+		if avd.Spec.PlatformRef == platform && slices.Contains(tenant.Spec.DomainRefs, avd.Spec.DomainRef) {
+			err := applyTenantOverrides(avd, tenant.Name)
+			if err != nil {
+				return err
+			}
+
+			azureVirtualDesktops[n] = avd
 			n++
 		}
 	}
@@ -738,3 +767,95 @@ func getAzureVirtualDesktopPlatform(azureVirtualDesktop *provisioningv1.AzureVir
 
 	return platform, true
 }
+
+type ProvisioningResource interface {
+	*provisioningv1.AzureDatabase | *provisioningv1.AzureManagedDatabase | *provisioningv1.HelmRelease | *provisioningv1.AzureVirtualMachine | *provisioningv1.AzureVirtualDesktop
+
+	GetProvisioningMeta() *provisioningv1.ProvisioningMeta
+	GetSpec() any
+}
+
+func applyTenantOverrides[T ProvisioningResource](target T, tenantName string) error {
+	if target == nil {
+		return nil
+	}
+
+	overrides := target.GetProvisioningMeta().TenantOverrides
+
+	if overrides == nil {
+		return nil
+	}
+
+	tenantOverridesJson, exists := overrides[tenantName]
+	if !exists {
+		return nil
+	}
+
+	var tenantOverridesMap map[string]any
+	if err := json.Unmarshal(tenantOverridesJson.Raw, &tenantOverridesMap); err != nil {
+		return err
+	}
+
+	targetSpec := target.GetSpec()
+
+	targetSpecJsonBytes, err := json.Marshal(targetSpec)
+	if err != nil {
+		return err
+	}
+
+	var targetSpecMap map[string]any
+	if err := json.Unmarshal(targetSpecJsonBytes, &targetSpecMap); err != nil {
+		return err
+	}
+
+	if err := mergo.Merge(&targetSpecMap, tenantOverridesMap, mergo.WithOverride, mergo.WithTransformers(jsonTransformer{})); err != nil {
+		return err
+	}
+
+	targetSpecJsonBytes, err = json.Marshal(targetSpecMap)
+	if err != nil {
+		return err
+	}
+
+	if err := json.Unmarshal(targetSpecJsonBytes, targetSpec); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+type jsonTransformer struct {
+}
+
+func (t jsonTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
+	if typ == reflect.TypeOf(apiextensionsv1.JSON{}) {
+		return func(dst, src reflect.Value) error {
+			if dst.CanSet() {
+				srcRaw := src.FieldByName("Raw").Bytes()
+				var srcMap map[string]interface{}
+				if err := json.Unmarshal(srcRaw, &srcMap); err != nil {
+					return err
+				}
+
+				dstRaw := dst.FieldByName("Raw").Bytes()
+				var dstMap map[string]interface{}
+				if err := json.Unmarshal(dstRaw, &dstMap); err != nil {
+					return err
+				}
+
+				if err := mergo.Merge(&dstMap, srcMap, mergo.WithOverride, mergo.WithTransformers(jsonTransformer{})); err != nil {
+					return err
+				}
+
+				dstRaw, err := json.Marshal(dstMap)
+				if err != nil {
+					return err
+				}
+
+				dst.FieldByName("Raw").SetBytes(dstRaw)
+			}
+			return nil
+		}
+	}
+	return nil
+}