Allow Resources to be Optional in Pipeline

Pipeline inputs and outputs are considered required, there is no way
today to mark them optional. This change introduces a new field called
optional as part of the PipelineTaskInputResource and PipelineTaskOutputResource
by default a resource is required. To mark any resource optional, set optional to true:

apiVersion: tekton.dev/v1alpha1
kind: Pipeline
metadata:
 name: pipeline-build-image
spec:
 inputs:
   resources:
     - name: workspace
       type: git
       optional: true
 tasks:
   - name: check-workspace

Closes tektoncd#1710

examples/pipelineruns/demo-optional-resources.yaml

@@ -0,0 +1,130 @@
+apiVersion: tekton.dev/v1alpha1
+kind: Condition
+metadata:
+  name: check-git-pipeline-resource
+spec:
+  params:
+    - name: "path"
+  resources:
+    - name: git-repo
+      type: git
+      optional: true
+  check:
+    image: alpine
+    command: ["/bin/sh"]
+    args: ['-c', 'test -f $(resources.git-repo.path)/$(params.path)']
+---
+
+apiVersion: tekton.dev/v1alpha1
+kind: Condition
+metadata:
+  name: check-image-pipeline-resource
+spec:
+  resources:
+    - name: built-image
+      type: image
+      optional: true
+  check:
+    image: alpine
+    command: ["/bin/sh"]
+    args: ['-c', 'test ! -z $(resources.built-image.url)']
+---
+
+apiVersion: tekton.dev/v1alpha1
+kind: Task
+metadata:
+  name: build-an-image
+spec:
+  inputs:
+    resources:
+      - name: git-repo
+        type: git
+        optional: true
+    params:
+      - name: DOCKERFILE
+        description: The path to the dockerfile to build from GitHub Repo
+        default: "Dockerfile"
+  outputs:
+    resources:
+      - name: built-image
+        type: image
+        optional: true
+  steps:
+    - name: build-an-image
+      image: "gcr.io/kaniko-project/executor:latest"
+      command:
+        - /kaniko/executor
+      args:
+        - --dockerfile=$(inputs.params.DOCKERFILE)
+        - --destination=$(outputs.resources.built-image.url)
+---
+
+apiVersion: tekton.dev/v1alpha1
+kind: Pipeline
+metadata:
+  name: demo-pipeline-to-build-an-image
+spec:
+  resources:
+    - name: source-repo
+      type: git
+      optional: true
+    - name: web-image
+      type: image
+      optional: true
+  params:
+    - name: "path"
+      default: "README.md"
+  tasks:
+    - name: build-an-image
+      taskRef:
+        name: build-an-image
+      conditions:
+        - conditionRef: "check-git-pipeline-resource"
+          params:
+            - name: "path"
+              value: "$(params.path)"
+          resources:
+            - name: git-repo
+              resource: source-repo
+        - conditionRef: "check-image-pipeline-resource"
+          resources:
+            - name: built-image
+              resource: web-image
+      resources:
+        inputs:
+          - name: git-repo
+            resource: source-repo
+        outputs:
+          - name: built-image
+            resource: web-image
+
+---
+
+apiVersion: tekton.dev/v1alpha1
+kind: PipelineRun
+metadata:
+  name: demo-pipeline-run-1
+spec:
+  pipelineRef:
+    name: demo-pipeline-to-build-an-image
+  serviceAccountName: 'default'
+---
+
+apiVersion: tekton.dev/v1alpha1
+kind: PipelineRun
+metadata:
+  name: demo-pipeline-run-2
+spec:
+  pipelineRef:
+    name: demo-pipeline-to-build-an-image
+  serviceAccountName: 'default'
+  resources:
+    - name: source-repo
+      resourceSpec:
+        type: git
+        params:
+          - name: revision
+            value: master
+          - name: url
+            value: https://github.com/tektoncd/pipeline
+---

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	cloud.google.com/go v0.47.0 // indirect
+	cloud.google.com/go/storage v1.0.0
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0 // indirect
 	contrib.go.opencensus.io/exporter/stackdriver v0.12.8 // indirect
 	github.com/Azure/azure-sdk-for-go v36.1.0+incompatible // indirect
@@ -14,6 +15,7 @@ require (
 	github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20191203181535-308b93ad1f39
 	github.com/aws/aws-sdk-go v1.25.31 // indirect
 	github.com/cloudevents/sdk-go v0.0.0-20190509003705-56931988abe3
+	github.com/davecgh/go-spew v1.1.1
 	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-openapi/jsonreference v0.19.3 // indirect
@@ -51,7 +53,7 @@ require (
 	golang.org/x/sys v0.0.0-20191119060738-e882bf8e40c2 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
 	golang.org/x/tools v0.0.0-20191118222007-07fc4c7f2b98 // indirect
-	google.golang.org/api v0.10.0 // indirect
+	google.golang.org/api v0.10.0
 	google.golang.org/appengine v1.6.5 // indirect
 	google.golang.org/grpc v1.24.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

pkg/apis/pipeline/v1alpha1/pipeline_types.go

@@ -174,6 +174,10 @@ type PipelineDeclaredResource struct {
 	Name string `json:"name"`
 	// Type is the type of the PipelineResource.
 	Type PipelineResourceType `json:"type"`
+	// Optional declares the resource as optional.
+	// optional: true - the resource is considered optional
+	// optional: false - the resource is considered required (default/equivalent of not specifying it)
+	Optional bool `json:"optional,omitempty"`
 }
 
 // PipelineConditionResource allows a Pipeline to declare how its DeclaredPipelineResources

pkg/reconciler/pipelinerun/resources/conditionresolution.go

@@ -19,7 +19,6 @@ package resources
 
 import (
 	"fmt"
-
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline"
 	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
 	corev1 "k8s.io/api/core/v1"

pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go

@@ -18,6 +18,7 @@ package resources
 
 import (
 	"fmt"
+	"github.com/davecgh/go-spew/spew"
 	"reflect"
 
 	"go.uber.org/zap"
@@ -187,15 +188,29 @@ func GetResourcesFromBindings(pr *v1alpha1.PipelineRun, getResource resources.Ge
 // ValidateResourceBindings validate that the PipelineResources declared in Pipeline p are bound in PipelineRun.
 func ValidateResourceBindings(p *v1alpha1.PipelineSpec, pr *v1alpha1.PipelineRun) error {
 	required := make([]string, 0, len(p.Resources))
+	optional := make([]string, 0, len(p.Resources))
 	for _, resource := range p.Resources {
-		required = append(required, resource.Name)
+		if resource.Optional {
+			// create a list of optional resources
+			optional = append(optional, resource.Name)
+		} else {
+			// create a list of required resources
+			required = append(required, resource.Name)
+		}
 	}
 	provided := make([]string, 0, len(pr.Spec.Resources))
 	for _, resource := range pr.Spec.Resources {
 		provided = append(provided, resource.Name)
 	}
-	if err := list.IsSame(required, provided); err != nil {
-		return fmt.Errorf("pipelineRun bound resources didn't match Pipeline: %w", err)
+	// verify that the list of required resources exists in the provided resources
+	missing := list.DiffLeft(required, provided)
+	if len(missing) > 0 {
+		return fmt.Errorf("Pipeline's declared required resources are missing from the PipelineRun: %s", missing)
+	}
+	// verify that the list of provided resources does not have any extra resources (outside of required and optional resources combined)
+	extra := list.DiffLeft(provided, append(required, optional...))
+	if len(extra) > 0 {
+		return fmt.Errorf("PipelineRun's declared resources didn't match usage in Pipeline: %s", extra)
 	}
 	return nil
 }
@@ -426,11 +441,15 @@ func resolveConditionChecks(pt *v1alpha1.PipelineTask, taskRunStatus map[string]
 		}
 		conditionResources := map[string]*v1alpha1.PipelineResource{}
 		for _, declared := range ptc.Resources {
-			r, ok := providedResources[declared.Resource]
-			if !ok {
-				return nil, fmt.Errorf("resources %s missing for condition %s in pipeline task %s", declared.Resource, cName, pt.Name)
+			if r, ok := providedResources[declared.Resource]; ok {
+				conditionResources[declared.Name] = r
+			} else {
+				for _, resource := range c.Spec.Resources {
+					if declared.Name == resource.Name && !resource.Optional {
+						return nil, fmt.Errorf("resources %s missing for condition %s in pipeline task %s", declared.Resource, cName, pt.Name)
+					}
+				}
 			}
-			conditionResources[declared.Name] = r
 		}
 
 		rcc := ResolvedConditionCheck{
@@ -457,19 +476,37 @@ func ResolvePipelineTaskResources(pt v1alpha1.PipelineTask, ts *v1alpha1.TaskSpe
 		Outputs:  map[string]*v1alpha1.PipelineResource{},
 	}
 	if pt.Resources != nil {
+		spew.Dump(pt.Resources.Inputs)
+		spew.Dump(ts)
 		for _, taskInput := range pt.Resources.Inputs {
-			resource, ok := providedResources[taskInput.Resource]
-			if !ok {
-				return nil, fmt.Errorf("pipelineTask tried to use input resource %s not present in declared resources", taskInput.Resource)
+			if resource, ok := providedResources[taskInput.Resource]; ok {
+				rtr.Inputs[taskInput.Name] = resource
+			} else {
+				if ts.Inputs == nil {
+					return nil, fmt.Errorf("pipelineTask tried to use input resource %s not present in declared resources", taskInput.Resource)
+				} else {
+					for _, r := range ts.Inputs.Resources {
+						if r.Name == taskInput.Name && !r.Optional {
+							return nil, fmt.Errorf("pipelineTask tried to use input resource %s not present in declared resources", taskInput.Resource)
+						}
+					}
+				}
 			}
-			rtr.Inputs[taskInput.Name] = resource
 		}
 		for _, taskOutput := range pt.Resources.Outputs {
-			resource, ok := providedResources[taskOutput.Resource]
-			if !ok {
-				return nil, fmt.Errorf("pipelineTask tried to use output resource %s not present in declared resources", taskOutput.Resource)
+			if resource, ok := providedResources[taskOutput.Resource]; ok {
+				rtr.Outputs[taskOutput.Name] = resource
+			} else {
+				if ts.Outputs == nil {
+					return nil, fmt.Errorf("pipelineTask tried to use output resource %s not present in declared resources", taskOutput.Resource)
+				} else {
+					for _, r := range ts.Outputs.Resources {
+						if r.Name == taskOutput.Name && !r.Optional {
+							return nil, fmt.Errorf("pipelineTask tried to use output resource %s not present in declared resources", taskOutput.Resource)
+						}
+					}
+				}
 			}
-			rtr.Outputs[taskOutput.Name] = resource
 		}
 	}
 	return &rtr, nil