Add pipeline artifact config to the share store

The pipeline controller loads the artifact storage config maps via
the client on every reconcile. Refactor the artifact storage code
to extract the configuration aspects and move them into the shared
storage.

Update the artifact storage to use config from the store.
Since the artifact storage interface changed as a consequence,
update all the consumers accordingly: input and out resources
and the task run controller.

config/config-artifact-bucket.yaml

@@ -28,3 +28,6 @@ metadata:
 #    bucket.service.account.secret.name:
 #    # The key in the secret with the required service account json
 #    bucket.service.account.secret.key:
+#    # The field name that should be used for the service account
+#    # Valid values: GOOGLE_APPLICATION_CREDENTIALS, BOTO_CONFIG.
+#    bucket.service.account.field.name: GOOGLE_APPLICATION_CREDENTIALS

pkg/apis/config/artifact_bucket.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2019 The Tekton Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"os"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	// bucketLocationKey is the name of the configmap entry that specifies
+	// loction of the bucket.
+	BucketLocationKey = "location"
+
+	// BucketServiceAccountSecretNameKey is the name of the configmap entry that specifies
+	// the name of the secret that will provide the servie account with bucket access.
+	// This secret must  have a key called serviceaccount that will have a value with
+	// the service account with access to the bucket
+	BucketServiceAccountSecretNameKey = "bucket.service.account.secret.name"
+
+	// BucketServiceAccountSecretKeyKey is the name of the configmap entry that specifies
+	// the secret key that will have a value with the service account json with access
+	// to the bucket
+	BucketServiceAccountSecretKeyKey = "bucket.service.account.secret.key"
+
+	// DefaultBucketServiceAccountSecretKey defaults to a gcs bucket
+	DefaultBucketServiceFieldName = "GOOGLE_APPLICATION_CREDENTIALS"
+
+	// bucketServiceAccountFieldName is the name of the configmap entry that specifies
+	// the field name that should be used for the service account.
+	// Valid values: GOOGLE_APPLICATION_CREDENTIALS, BOTO_CONFIG.
+	BucketServiceAccountFieldNameKey = "bucket.service.account.field.name"
+)
+
+// ArtifactPVC holds the configurations for the artifacts PVC
+// +k8s:deepcopy-gen=true
+type ArtifactBucket struct {
+	Location                 string
+	ServiceAccountSecretName string
+	ServiceAccountSecretKey  string
+	ServiceAccountFieldName  string
+}
+
+// GetArtifactBucketConfigName returns the name of the configmap containing all
+// customizations for the storage bucket.
+func GetArtifactBucketConfigName() string {
+	if e := os.Getenv("CONFIG_ARTIFACT_BUCKET_NAME"); e != "" {
+		return e
+	}
+	return "config-artifact-bucket"
+}
+
+// Equals returns true if two Configs are identical
+func (cfg *ArtifactBucket) Equals(other *ArtifactBucket) bool {
+	if cfg == nil && other == nil {
+		return true
+	}
+
+	if cfg == nil || other == nil {
+		return false
+	}
+
+	return other.Location == cfg.Location &&
+		other.ServiceAccountSecretName == cfg.ServiceAccountSecretName &&
+		other.ServiceAccountSecretKey == cfg.ServiceAccountSecretKey &&
+		other.ServiceAccountFieldName == cfg.ServiceAccountFieldName
+}
+
+// NewArtifactBucketFromMap returns a Config given a map corresponding to a ConfigMap
+func NewArtifactBucketFromMap(cfgMap map[string]string) (*ArtifactBucket, error) {
+	tc := ArtifactBucket{
+		ServiceAccountFieldName: DefaultBucketServiceFieldName,
+	}
+
+	if location, ok := cfgMap[BucketLocationKey]; ok {
+		tc.Location = location
+	}
+
+	if serviceAccountSecretName, ok := cfgMap[BucketServiceAccountSecretNameKey]; ok {
+		tc.ServiceAccountSecretName = serviceAccountSecretName
+	}
+
+	if serviceAccountSecretKey, ok := cfgMap[BucketServiceAccountSecretKeyKey]; ok {
+		tc.ServiceAccountSecretKey = serviceAccountSecretKey
+	}
+
+	if serviceAccountFieldName, ok := cfgMap[BucketServiceAccountFieldNameKey]; ok {
+		tc.ServiceAccountFieldName = serviceAccountFieldName
+	}
+
+	return &tc, nil
+}
+
+// NewArtifactBucketFromConfigMap returns a Config for the given configmap
+func NewArtifactBucketFromConfigMap(config *corev1.ConfigMap) (*ArtifactBucket, error) {
+	return NewArtifactBucketFromMap(config.Data)
+}

pkg/apis/config/artifact_bucket_test.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2020 The Tekton Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/tektoncd/pipeline/pkg/apis/config"
+	test "github.com/tektoncd/pipeline/pkg/reconciler/testing"
+	"github.com/tektoncd/pipeline/test/diff"
+)
+
+func TestNewArtifactBucketFromConfigMap(t *testing.T) {
+	type testCase struct {
+		expectedConfig *config.ArtifactBucket
+		fileName       string
+	}
+
+	testCases := []testCase{
+		{
+			expectedConfig: &config.ArtifactBucket{
+				Location:                "gs://my-bucket",
+				ServiceAccountFieldName: "GOOGLE_APPLICATION_CREDENTIALS",
+			},
+			fileName: config.GetArtifactBucketConfigName(),
+		},
+		{
+			expectedConfig: &config.ArtifactBucket{
+				Location:                 "gs://test-bucket",
+				ServiceAccountSecretName: "test-secret",
+				ServiceAccountSecretKey:  "key",
+				ServiceAccountFieldName:  "some-field",
+			},
+			fileName: "config-artifact-bucket-all-set",
+		},
+	}
+
+	for _, tc := range testCases {
+		verifyConfigFileWithExpectedArtifactBucketConfig(t, tc.fileName, tc.expectedConfig)
+	}
+}
+
+func TestNewArtifactBucketFromEmptyConfigMap(t *testing.T) {
+	ArtifactBucketConfigEmptyName := "config-artifact-bucket-empty"
+	expectedConfig := &config.ArtifactBucket{
+		ServiceAccountFieldName: "GOOGLE_APPLICATION_CREDENTIALS",
+	}
+	verifyConfigFileWithExpectedArtifactBucketConfig(t, ArtifactBucketConfigEmptyName, expectedConfig)
+}
+
+func TestGetArtifactBucketConfigName(t *testing.T) {
+	for _, tc := range []struct {
+		description             string
+		artifactsBucketEnvValue string
+		expected                string
+	}{{
+		description:             "Artifact Bucket config value not set",
+		artifactsBucketEnvValue: "",
+		expected:                "config-artifact-bucket",
+	}, {
+		description:             "Artifact Bucket config value set",
+		artifactsBucketEnvValue: "config-artifact-bucket-test",
+		expected:                "config-artifact-bucket-test",
+	}} {
+		t.Run(tc.description, func(t *testing.T) {
+			original := os.Getenv("CONFIG_ARTIFACT_BUCKET_NAME")
+			defer t.Cleanup(func() {
+				os.Setenv("CONFIG_ARTIFACT_BUCKET_NAME", original)
+			})
+			if tc.artifactsBucketEnvValue != "" {
+				os.Setenv("CONFIG_ARTIFACT_BUCKET_NAME", tc.artifactsBucketEnvValue)
+			}
+			got := config.GetArtifactBucketConfigName()
+			want := tc.expected
+			if got != want {
+				t.Errorf("GetArtifactBucketConfigName() = %s, want %s", got, want)
+			}
+		})
+	}
+}
+
+func verifyConfigFileWithExpectedArtifactBucketConfig(t *testing.T, fileName string, expectedConfig *config.ArtifactBucket) {
+	cm := test.ConfigMapFromTestFile(t, fileName)
+	if ab, err := config.NewArtifactBucketFromConfigMap(cm); err == nil {
+		if d := cmp.Diff(ab, expectedConfig); d != "" {
+			t.Errorf("Diff:\n%s", diff.PrintWantGot(d))
+		}
+	} else {
+		t.Errorf("NewArtifactBucketFromConfigMap(actual) = %v", err)
+	}
+}

pkg/apis/config/artifact_pvc.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2019 The Tekton Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"os"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	// DefaultPVCSize is the default size of the PVC to create
+	DefaultPVCSize = "5Gi"
+
+	// PVCSizeKey is the name of the configmap entry that specifies the size of the PVC to create
+	PVCSizeKey = "size"
+
+	// PVCStorageClassNameKey is the name of the configmap entry that specifies the storage class of the PVC to create
+	PVCStorageClassNameKey = "storageClassName"
+)
+
+// ArtifactPVC holds the configurations for the artifacts PVC
+// +k8s:deepcopy-gen=true
+type ArtifactPVC struct {
+	Size             string
+	StorageClassName string
+}
+
+// GetArtifactPVCConfigName returns the name of the configmap containing all
+// customizations for the storage PVC.
+func GetArtifactPVCConfigName() string {
+	if e := os.Getenv("CONFIG_ARTIFACT_PVC_NAME"); e != "" {
+		return e
+	}
+	return "config-artifact-pvc"
+}
+
+// Equals returns true if two Configs are identical
+func (cfg *ArtifactPVC) Equals(other *ArtifactPVC) bool {
+	if cfg == nil && other == nil {
+		return true
+	}
+
+	if cfg == nil || other == nil {
+		return false
+	}
+
+	return other.Size == cfg.Size &&
+		other.StorageClassName == cfg.StorageClassName
+}
+
+// NewDefaultsFromMap returns a Config given a map corresponding to a ConfigMap
+func NewArtifactPVCFromMap(cfgMap map[string]string) (*ArtifactPVC, error) {
+	tc := ArtifactPVC{
+		Size: DefaultPVCSize,
+	}
+
+	if size, ok := cfgMap[PVCSizeKey]; ok {
+		tc.Size = size
+	}
+
+	if storageClassName, ok := cfgMap[PVCStorageClassNameKey]; ok {
+		tc.StorageClassName = storageClassName
+	}
+
+	return &tc, nil
+}
+
+// NewDefaultsFromConfigMap returns a Config for the given configmap
+func NewArtifactPVCFromConfigMap(config *corev1.ConfigMap) (*ArtifactPVC, error) {
+	return NewArtifactPVCFromMap(config.Data)
+}