Add secret and configmap support as workspaces

- Handle multiple types of workspaces
- Handle Secret and ConfigMap volumes with workspaces
- New `files` package to move all "on-the-fly" file creation in one
  place.

Signed-off-by: Vincent Demeester <[email protected]>

cmd/tkn-local/run.go

@@ -112,7 +112,6 @@ func run(opts *runOption) error {
 			"context":    dir,
 			"dockerfile": dir,
 		},
-		// FrontAttrs is handled after
 		Session: attachable,
 		// CacheExports: c.cfg.CacheExports,
 		// CacheImports: c.cfg.CacheImports,

examples/1-pipelinerun-with-workspaces/run.yaml

@@ -0,0 +1,135 @@
+# In this contrived example 3 different kinds of workspace volume are used to thread
+# data through a pipeline's tasks.
+# 1. A ConfigMap is used as source of recipe data.
+# 2. A Secret is used to store a password.
+# 3. A PVC is used to share data from one task to the next.
+#
+# The end result is a pipeline that first checks if the password is correct and, if so,
+# copies data out of a recipe store onto a shared volume. The recipe data is then read
+# by a subsequent task and printed to screen.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sensitive-recipe-storage
+data:
+  brownies: |
+    1. Heat oven to 325 degrees F
+    2. Melt 1/2 cup butter w/ 1/2 cup cocoa, stirring smooth.
+    3. Remove from heat, allow to cool for a few minutes.
+    4. Transfer to bowl.
+    5. Whisk in 2 eggs, one at a time.
+    6. Stir in vanilla.
+    7. Separately combine 1 cup sugar, 1/4 cup flour, 1 cup chopped
+       walnuts and pinch of salt
+    8. Combine mixtures.
+    9. Bake in greased pan for 30 minutes. Watch carefully for
+       appropriate level of gooeyness.
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: secret-password
+type: Opaque
+data:
+  password: aHVudGVyMg==
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: shared-task-storage
+spec:
+  resources:
+    requests:
+      storage: 16Mi
+  volumeMode: Filesystem
+  accessModes:
+    - ReadWriteOnce
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: fetch-secure-data
+spec:
+  workspaces:
+  - name: super-secret-password
+  - name: secure-store
+  - name: filedrop
+  steps:
+  - name: fetch-and-write
+    image: ubuntu
+    script: |
+      if [ "hunter2" = "$(cat $(workspaces.super-secret-password.path)/password)" ]; then
+        cp $(workspaces.secure-store.path)/recipe.txt $(workspaces.filedrop.path)
+      else
+        echo "wrong password!"
+        exit 1
+      fi
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: print-data
+spec:
+  workspaces:
+  - name: storage
+    readOnly: true
+  params:
+  - name: filename
+  steps:
+  - name: print-secrets
+    image: ubuntu
+    script: cat $(workspaces.storage.path)/$(params.filename)
+---
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  name: fetch-and-print-recipe
+spec:
+  workspaces:
+  - name: password-vault
+  - name: recipe-store
+  - name: shared-data
+  tasks:
+  - name: fetch-the-recipe
+    taskRef:
+      name: fetch-secure-data
+    workspaces:
+    - name: super-secret-password
+      workspace: password-vault
+    - name: secure-store
+      workspace: recipe-store
+    - name: filedrop
+      workspace: shared-data
+  - name: print-the-recipe
+    taskRef:
+      name: print-data
+    # Note: this is currently required to ensure order of write / read on PVC is correct.
+    runAfter:
+      - fetch-the-recipe
+    params:
+    - name: filename
+      value: recipe.txt
+    workspaces:
+    - name: storage
+      workspace: shared-data
+---
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  generateName: recipe-time-
+spec:
+  pipelineRef:
+    name: fetch-and-print-recipe
+  workspaces:
+  - name: password-vault
+    secret:
+      secretName: secret-password
+  - name: recipe-store
+    configMap:
+      name: sensitive-recipe-storage
+      items:
+      - key: brownies
+        path: recipe.txt
+  - name: shared-data
+    persistentVolumeClaim:
+      claimName: shared-task-storage

pkg/tekton/files/config.go

@@ -0,0 +1,32 @@
+package files
+
+import (
+	"github.com/moby/buildkit/client/llb"
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func ConfigMap(configmap *corev1.ConfigMap, configmapSource *corev1.ConfigMapVolumeSource) (llb.State, error) {
+	state := llb.Scratch().Dir("/")
+	if len(configmapSource.Items) == 0 {
+		for name, value := range configmap.Data {
+			state = addConfigMap(state, configmap.Name, name, value)
+		}
+	} else {
+		for _, item := range configmapSource.Items {
+			value, ok := configmap.Data[item.Key]
+			if !ok {
+				return llb.State{}, errors.Errorf("key %s from configmap %s not found in context", item.Key, configmap.Name)
+			}
+			state = addConfigMap(state, configmap.Name, item.Path, value)
+		}
+	}
+	return state, nil
+}
+
+func addConfigMap(state llb.State, configmapName, name, value string) llb.State {
+	return state.File(
+		llb.Mkfile(name, 0755, []byte(value)),
+		llb.WithCustomName("[tekton] configmap "+configmapName+"/"+name+": preparing file"),
+	)
+}

pkg/tekton/files/config_test.go

@@ -0,0 +1,70 @@
+package files_test
+
+import (
+	"testing"
+
+	"github.com/vdemeester/buildkit-tekton/pkg/tekton/files"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestConfigMapMissingItem(t *testing.T) {
+	configmap := &corev1.ConfigMap{
+		Data: map[string]string{
+			"configmap": "value",
+		},
+	}
+	configmapSource := &corev1.ConfigMapVolumeSource{
+		Items: []corev1.KeyToPath{{
+			Key:  "notfound",
+			Path: "foo.txt",
+		}},
+	}
+	_, err := files.ConfigMap(configmap, configmapSource)
+	if err == nil {
+		t.Fatalf("expected an error, got nothing")
+	} else {
+	}
+}
+
+func TestConfigMap(t *testing.T) {
+	tests := []struct {
+		name            string
+		configmap       *corev1.ConfigMap
+		configmapSource *corev1.ConfigMapVolumeSource
+	}{{
+		name: "all-keys",
+		configmap: &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{Name: "myconfigmap"},
+			Data: map[string]string{
+				"configmap1": "value1",
+				"configmap2": "value2",
+			},
+		},
+		configmapSource: &corev1.ConfigMapVolumeSource{},
+	}, {
+		name: "with-items",
+		configmap: &corev1.ConfigMap{
+			Data: map[string]string{
+				"configmap1": "value1",
+				"configmap2": "value2",
+			},
+		},
+		configmapSource: &corev1.ConfigMapVolumeSource{
+			Items: []corev1.KeyToPath{{
+				Key:  "configmap1",
+				Path: "foo.txt",
+			}},
+		},
+	}}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := files.ConfigMap(tc.configmap, tc.configmapSource)
+			if err != nil {
+				t.Fatal(err)
+			}
+			// FIXME(vdemeester) exercise this better, most likely using buildkit testutil (integration)
+		})
+	}
+}

pkg/tekton/files/script.go

@@ -0,0 +1,30 @@
+package files
+
+import (
+	"strings"
+
+	"github.com/moby/buildkit/client/llb"
+	"github.com/tektoncd/pipeline/pkg/names"
+)
+
+const (
+	defaultScriptPreamble = "#!/bin/sh\nset -e\n"
+)
+
+func Script(stepName, scriptName, script string) (string, llb.State) {
+	// Check for a shebang, and add a default if it's not set.
+	// The shebang must be the first non-empty line.
+	cleaned := strings.TrimSpace(script)
+	hasShebang := strings.HasPrefix(cleaned, "#!")
+
+	if !hasShebang {
+		script = defaultScriptPreamble + script
+	}
+	filename := names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(scriptName)
+	data := script
+	scriptSt := llb.Scratch().Dir("/").File(
+		llb.Mkfile(filename, 0755, []byte(data)),
+		llb.WithCustomName("[tekton] "+stepName+": preparing script"),
+	)
+	return filename, scriptSt
+}

pkg/tekton/files/script_test.go

@@ -0,0 +1,35 @@
+package files_test
+
+import (
+	"testing"
+
+	"github.com/vdemeester/buildkit-tekton/pkg/tekton/files"
+)
+
+func TestScript(t *testing.T) {
+	tests := []struct {
+		name, script, expected string
+	}{{
+		name: "no-shebang",
+		script: `echo hello world
+cat foo`,
+		expected: `#!/bin/sh
+set -e
+echo hello world
+cat foo`,
+	}, {
+		name: "with shebang",
+		script: `#!/usr/bin/env bash
+echo foo`,
+		expected: `#!/usr/bin/env bash
+echo foo`,
+	}}
+	for _, tc := range tests {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			filename, state := files.Script("stepName", "scriptName", tc.script)
+			// FIXME(vdemeester) exercise this better, most likely using buildkit testutil (integration)
+			t.Logf("%s: %+v", filename, state)
+		})
+	}
+}

pkg/tekton/files/secret.go

@@ -0,0 +1,32 @@
+package files
+
+import (
+	"github.com/moby/buildkit/client/llb"
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func Secret(secret *corev1.Secret, secretSource *corev1.SecretVolumeSource) (llb.State, error) {
+	state := llb.Scratch().Dir("/")
+	if len(secretSource.Items) == 0 {
+		for name, value := range secret.Data {
+			state = addSecret(state, secret.Name, name, value)
+		}
+	} else {
+		for _, item := range secretSource.Items {
+			value, ok := secret.Data[item.Key]
+			if !ok {
+				return llb.State{}, errors.Errorf("key %s from secret %s not found in context", item.Key, secret.Name)
+			}
+			state = addSecret(state, secret.Name, item.Path, value)
+		}
+	}
+	return state, nil
+}
+
+func addSecret(state llb.State, secretName, name string, value []byte) llb.State {
+	return state.File(
+		llb.Mkfile(name, 0755, value),
+		llb.WithCustomName("[tekton] secret "+secretName+"/"+name+": preparing file"),
+	)
+}