From 5938dc88964eb9f07ce5984fee0ceccddb2ea03e Mon Sep 17 00:00:00 2001 From: Shivam Mukhade Date: Fri, 24 Sep 2021 12:41:01 +0530 Subject: [PATCH] [OpenShift] Updates pipelines extension to use InstallerSet This updates the extension to use installer set for pre and post reconciler, this will allow change of targetnamespace now previously the target namespace for extension resources was hardcoded to openshift-pipelines and was not possible to change. Installer set will allow this now. Signed-off-by: Shivam Mukhade --- .../operator/v1alpha1/tektonpipeline_types.go | 6 +- .../v1alpha1/zz_generated.deepcopy.go | 7 + .../kubernetes/tektoninstallerset/install.go | 2 + .../openshift/tektonpipeline/extension.go | 117 +++++++++---- .../openshift/tektonpipeline/installerset.go | 160 ++++++++++++++++++ 5 files changed, 259 insertions(+), 33 deletions(-) create mode 100644 pkg/reconciler/openshift/tektonpipeline/installerset.go diff --git a/pkg/apis/operator/v1alpha1/tektonpipeline_types.go b/pkg/apis/operator/v1alpha1/tektonpipeline_types.go index 52b9a6b177..c9041de700 100644 --- a/pkg/apis/operator/v1alpha1/tektonpipeline_types.go +++ b/pkg/apis/operator/v1alpha1/tektonpipeline_types.go @@ -58,9 +58,13 @@ type TektonPipelineStatus struct { // +optional Version string `json:"version,omitempty"` - // The current installer set name + // The current installer set name for TektonPipeline // +optional TektonInstallerSet string `json:"tektonInstallerSet,omitempty"` + + // The installer sets created for extension components + // +optional + ExtentionInstallerSets map[string]string `json:"extTektonInstallerSets,omitempty"` } // TektonPipelineList contains a list of TektonPipeline diff --git a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go index fa0f031415..f5e17723d2 100644 --- a/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/operator/v1alpha1/zz_generated.deepcopy.go @@ -810,6 +810,13 @@ func (in *TektonPipelineSpec) DeepCopy() *TektonPipelineSpec { func (in *TektonPipelineStatus) DeepCopyInto(out *TektonPipelineStatus) { *out = *in in.Status.DeepCopyInto(&out.Status) + if in.ExtentionInstallerSets != nil { + in, out := &in.ExtentionInstallerSets, &out.ExtentionInstallerSets + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } return } diff --git a/pkg/reconciler/kubernetes/tektoninstallerset/install.go b/pkg/reconciler/kubernetes/tektoninstallerset/install.go index ca71b8cf6f..0daf027dc6 100644 --- a/pkg/reconciler/kubernetes/tektoninstallerset/install.go +++ b/pkg/reconciler/kubernetes/tektoninstallerset/install.go @@ -48,6 +48,7 @@ var ( pipelinePred = mf.ByKind("Pipeline") // OpenShift Specific + serviceMonitorPred = mf.ByKind("ServiceMonitor") consoleCLIDownloadPred = mf.ByKind("ConsoleCLIDownload") consoleQuickStartPred = mf.ByKind("ConsoleQuickStart") ConsoleYAMLSamplePred = mf.ByKind("ConsoleYAMLSample") @@ -95,6 +96,7 @@ func (i *installer) EnsureNamespaceScopedResources() error { secretPred, horizontalPodAutoscalerPred, pipelinePred, + serviceMonitorPred, )).Apply(); err != nil { return err } diff --git a/pkg/reconciler/openshift/tektonpipeline/extension.go b/pkg/reconciler/openshift/tektonpipeline/extension.go index d348ba7a53..fa879275f8 100644 --- a/pkg/reconciler/openshift/tektonpipeline/extension.go +++ b/pkg/reconciler/openshift/tektonpipeline/extension.go @@ -30,6 +30,8 @@ import ( "github.com/tektoncd/operator/pkg/reconciler/common" occommon "github.com/tektoncd/operator/pkg/reconciler/openshift/common" "go.uber.org/zap" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "knative.dev/pkg/injection" "knative.dev/pkg/logging" @@ -43,6 +45,11 @@ const ( enableMetricsKey = "enableMetrics" enableMetricsDefaultValue = "true" + + versionKey = "VERSION" + + prePipelineInstallerSet = "PrePipelineInstallerSet" + postPipelineInstallerSet = "PostPipelineInstallerSet" ) func OpenShiftExtension(ctx context.Context) common.Extension { @@ -56,9 +63,16 @@ func OpenShiftExtension(ctx context.Context) common.Extension { if err != nil { logger.Fatalw("error creating initial manifest", zap.Error(err)) } + + version := os.Getenv(versionKey) + if version == "" { + logger.Fatal("Failed to find version from env") + } + ext := openshiftExtension{ operatorClientSet: operatorclient.Get(ctx), manifest: manifest, + version: version, } return ext } @@ -66,6 +80,7 @@ func OpenShiftExtension(ctx context.Context) common.Extension { type openshiftExtension struct { operatorClientSet versioned.Interface manifest mf.Manifest + version string } func (oe openshiftExtension) Transformers(comp v1alpha1.TektonComponent) []mf.Transformer { @@ -84,33 +99,46 @@ func (oe openshiftExtension) Transformers(comp v1alpha1.TektonComponent) []mf.Tr return trns } -func (oe openshiftExtension) PreReconcile(ctx context.Context, tc v1alpha1.TektonComponent) error { +func (oe openshiftExtension) PreReconcile(ctx context.Context, comp v1alpha1.TektonComponent) error { koDataDir := os.Getenv(common.KoEnvKey) + tp := comp.(*v1alpha1.TektonPipeline) - // make sure that openshift-pipelines namespace exists - namespaceLocation := filepath.Join(koDataDir, "tekton-namespace") - if err := common.AppendManifest(&oe.manifest, namespaceLocation); err != nil { + exist, err := checkIfInstallerSetExist(ctx, oe.operatorClientSet, oe.version, tp, prePipelineInstallerSet) + if err != nil { return err } - // add inject CA bundles manifests - cabundlesLocation := filepath.Join(koDataDir, "cabundles") - if err := common.AppendManifest(&oe.manifest, cabundlesLocation); err != nil { - return err - } + // If installer set doesn't exist then create a new one + if !exist { - // add pipelines-scc - pipelinesSCCLocation := filepath.Join(koDataDir, "tekton-pipeline", "00-prereconcile") - if err := common.AppendManifest(&oe.manifest, pipelinesSCCLocation); err != nil { - return err - } + // make sure that openshift-pipelines namespace exists + namespaceLocation := filepath.Join(koDataDir, "tekton-namespace") + if err := common.AppendManifest(&oe.manifest, namespaceLocation); err != nil { + return err + } - // Apply the resources - if err := oe.manifest.Apply(); err != nil { - return err + // add inject CA bundles manifests + cabundlesLocation := filepath.Join(koDataDir, "cabundles") + if err := common.AppendManifest(&oe.manifest, cabundlesLocation); err != nil { + return err + } + + // add pipelines-scc + pipelinesSCCLocation := filepath.Join(koDataDir, "tekton-pipeline", "00-prereconcile") + if err := common.AppendManifest(&oe.manifest, pipelinesSCCLocation); err != nil { + return err + } + + if err := common.Transform(ctx, &oe.manifest, tp); err != nil { + return err + } + + if err := createInstallerSet(ctx, oe.operatorClientSet, tp, oe.manifest, oe.version, + prePipelineInstallerSet, "pre-pipeline"); err != nil { + return err + } } - tp := tc.(*v1alpha1.TektonPipeline) if crUpdated := SetDefault(&tp.Spec.Pipeline); crUpdated { if _, err := oe.operatorClientSet.OperatorV1alpha1().TektonPipelines().Update(ctx, tp, v1.UpdateOptions{}); err != nil { return err @@ -127,25 +155,50 @@ func (oe openshiftExtension) PostReconcile(ctx context.Context, comp v1alpha1.Te // Install monitoring if metrics is enabled value := findParam(pipeline.Spec.Params, enableMetricsKey) - monitoringLocation := filepath.Join(koDataDir, "openshift-monitoring") - if err := common.AppendManifest(&oe.manifest, monitoringLocation); err != nil { - return err + if value == "true" { + exist, err := checkIfInstallerSetExist(ctx, oe.operatorClientSet, oe.version, pipeline, postPipelineInstallerSet) + if err != nil { + return err + } + + if !exist { + + monitoringLocation := filepath.Join(koDataDir, "openshift-monitoring") + if err := common.AppendManifest(&oe.manifest, monitoringLocation); err != nil { + return err + } + + if err := common.Transform(ctx, &oe.manifest, pipeline); err != nil { + return err + } + + if err := createInstallerSet(ctx, oe.operatorClientSet, pipeline, oe.manifest, oe.version, + postPipelineInstallerSet, "post-pipeline"); err != nil { + return err + } + } + + } else { + return deleteInstallerSet(ctx, oe.operatorClientSet, pipeline, postPipelineInstallerSet) } - trns, err := oe.manifest.Transform( - mf.InjectNamespace(pipeline.Spec.TargetNamespace), - mf.InjectOwner(pipeline), - ) - if err != nil { - return err + return nil +} +func (oe openshiftExtension) Finalize(ctx context.Context, comp v1alpha1.TektonComponent) error { + pipeline := comp.(*v1alpha1.TektonPipeline) + + installerSets := pipeline.Status.ExtentionInstallerSets + if len(installerSets) == 0 { + return nil } - if value == "true" { - return trns.Apply() + for _, value := range installerSets { + err := oe.operatorClientSet.OperatorV1alpha1().TektonInstallerSets().Delete(ctx, value, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return err + } } - return trns.Delete() -} -func (oe openshiftExtension) Finalize(context.Context, v1alpha1.TektonComponent) error { + return nil } diff --git a/pkg/reconciler/openshift/tektonpipeline/installerset.go b/pkg/reconciler/openshift/tektonpipeline/installerset.go new file mode 100644 index 0000000000..34b8b0e459 --- /dev/null +++ b/pkg/reconciler/openshift/tektonpipeline/installerset.go @@ -0,0 +1,160 @@ +/* +Copyright 2021 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tektonpipeline + +import ( + "context" + "fmt" + + mf "github.com/manifestival/manifestival" + "github.com/tektoncd/operator/pkg/apis/operator/v1alpha1" + clientset "github.com/tektoncd/operator/pkg/client/clientset/versioned" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + createdByKey = "operator.tekton.dev/created-by" + createdByValue = "TektonPipeline" + releaseVersionKey = "operator.tekton.dev/release-version" + targetNamespaceKey = "operator.tekton.dev/target-namespace" +) + +// checkIfInstallerSetExist checks if installer set exists for a component and return true/false based on it +// and if installer set which already exist is of older version/ or if target namespace is different then it +// deletes and return false to create a new installer set +func checkIfInstallerSetExist(ctx context.Context, oc clientset.Interface, relVersion string, + tp *v1alpha1.TektonPipeline, component string) (bool, error) { + + // Check if installer set is already created + compInstallerSet, ok := tp.Status.ExtentionInstallerSets[component] + if !ok { + return false, nil + } + + if compInstallerSet != "" { + // if already created then check which version it is + ctIs, err := oc.OperatorV1alpha1().TektonInstallerSets(). + Get(ctx, compInstallerSet, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return false, nil + } + return false, err + } + + // Check release version and target namespace + // If anyone of this is not as expected then delete existing + // installer set and create a new one + + version, vOk := ctIs.Annotations[releaseVersionKey] + namespace, nOk := ctIs.Annotations[targetNamespaceKey] + + if vOk && nOk { + if version == relVersion && namespace == tp.Spec.TargetNamespace { + // if installer set already exist + // release version and target namespace is as expected + // then ignore and move on + return true, nil + } + } + + // release version/ target namespace doesn't exist or is different from expected + // deleted existing InstallerSet and create a new one + + err = oc.OperatorV1alpha1().TektonInstallerSets(). + Delete(ctx, compInstallerSet, metav1.DeleteOptions{}) + if err != nil { + return false, err + } + } + + return false, nil +} + +func createInstallerSet(ctx context.Context, oc clientset.Interface, tp *v1alpha1.TektonPipeline, + manifest mf.Manifest, releaseVersion, component, installerSetPrefix string) error { + + is := makeInstallerSet(tp, manifest, installerSetPrefix, releaseVersion) + + createdIs, err := oc.OperatorV1alpha1().TektonInstallerSets(). + Create(ctx, is, metav1.CreateOptions{}) + if err != nil { + return err + } + + if len(tp.Status.ExtentionInstallerSets) == 0 { + tp.Status.ExtentionInstallerSets = map[string]string{} + } + + // Update the status of pipeline with created installerSet name + tp.Status.ExtentionInstallerSets[component] = createdIs.Name + + _, err = oc.OperatorV1alpha1().TektonPipelines(). + UpdateStatus(ctx, tp, metav1.UpdateOptions{}) + if err != nil { + return err + } + + return nil +} + +func makeInstallerSet(tp *v1alpha1.TektonPipeline, manifest mf.Manifest, prefix, releaseVersion string) *v1alpha1.TektonInstallerSet { + ownerRef := *metav1.NewControllerRef(tp, tp.GetGroupVersionKind()) + return &v1alpha1.TektonInstallerSet{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s-", prefix), + Labels: map[string]string{ + createdByKey: createdByValue, + }, + Annotations: map[string]string{ + releaseVersionKey: releaseVersion, + targetNamespaceKey: tp.Spec.TargetNamespace, + }, + OwnerReferences: []metav1.OwnerReference{ownerRef}, + }, + Spec: v1alpha1.TektonInstallerSetSpec{ + Manifests: manifest.Resources(), + }, + } +} + +func deleteInstallerSet(ctx context.Context, oc clientset.Interface, ta *v1alpha1.TektonPipeline, component string) error { + + compInstallerSet, ok := ta.Status.ExtentionInstallerSets[component] + if !ok { + return nil + } + + if compInstallerSet != "" { + // delete the installer set + err := oc.OperatorV1alpha1().TektonInstallerSets(). + Delete(ctx, ta.Status.ExtentionInstallerSets[component], metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return err + } + + // clear the name of installer set from TektonPipeline status + delete(ta.Status.ExtentionInstallerSets, component) + _, err = oc.OperatorV1alpha1().TektonPipelines().UpdateStatus(ctx, ta, metav1.UpdateOptions{}) + if err != nil && !errors.IsNotFound(err) { + return err + } + } + + return nil +}