Add end-to-end compiler tests

- Regenerate all "Golden" YAML files with PipelineRun
- Dynamically generate E2E test cases from "Golden" YAML files
- Add e2e_test target to Makefile
- Generate new "Golden" log files from Tekton pipeline run
- Fix PendingDeprecationWarning: dsl.ContainerOp....` will be removed in future releases
  Use `dsl.ContainerOp.container....` instead.
- Fix UserWarning: Missing type name was inferred as ...

Resolves kubeflow#28

.travis.yml

@@ -7,15 +7,15 @@ matrix:
       install:
         - python3 -m pip install -e sdk/python
       script:
-        - make test
+        - make unit_test
     - name: "Unit tests, Python 3.7"
       language: python
       python: "3.7"
       env: TOXENV=py37
       install:
         - python3 -m pip install -e sdk/python
       script:
-        - make test
+        - make unit_test
     - name: "Progress report on compiling KFP DSL test scripts"
       language: python
       python: "3.7"

Makefile

@@ -21,7 +21,7 @@ export PATH := ${VIRTUAL_ENV}/bin:${PATH}
 
 .PHONY: help
 help: ## Display the Make targets
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | \
+	@grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
 		awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}'
 
 .PHONY: venv
@@ -31,10 +31,22 @@ $(VENV)/bin/activate: sdk/python/setup.py
 	pip install -e sdk/python
 	@touch $(VENV)/bin/activate
 
-.PHONY: test
-test: venv ## Run compiler unit tests
+.PHONY: unit_test
+unit_test: venv ## Run compiler unit tests
 	@sdk/python/tests/run_tests.sh
 
+.PHONY: e2e_test
+e2e_test: venv ## Run compiler end-to-end tests (requires kubectl and tkn CLI)
+	@which kubectl || (echo "Missing kubectl CLI" && exit 1)
+	@test -z "${KUBECONFIG}" && echo "KUBECONFIG not set" && exit 1 || echo "${KUBECONFIG}"
+	@kubectl version --short || (echo "Failed to access kubernetes cluster" && exit 1)
+	@which tkn && tkn version || (echo "Missing tkn CLI" && exit 1)
+	@sdk/python/tests/run_e2e_tests.sh
+
+.PHONY: test
+test: unit_test e2e_test ## Run compiler unit tests and end-to-end tests
+	@echo OK
+
 .PHONY: report
 report: ## Report compilation status of KFP testdata DSL scripts
 	@cd sdk/python/tests && ./test_kfp_samples.sh

sdk/FEATURES.md

@@ -191,7 +191,8 @@ kubectl get pods will show a 'Completed' pod when a sidecar exits successfully b
 
 An exit handler is a component that always executes, irrespective of success or failure, at the end of the pipeline.
 
-The finally syntax is now supported in tekton preview mode, the final tasks can be specified but not executed yet, implementation and execution of final tasks are tracked in separate PR [Tracking pull request #2661][exitHandler]
+The `finally` syntax is now supported in Tekton preview mode. The final tasks can be specified but not executed yet,
+implementation and execution of final tasks are tracked in separate PR [Tracking pull request #2661][exitHandler]
 
 
 <!-- Issue and PR links-->

sdk/python/README.md

@@ -192,14 +192,18 @@ A manual E2E test can be performed in the following manner:
     kubectl apply -f <pipeline.yaml>
     tkn pipeline start <pipeline-name> --showlog
 
-
 Some E2E tests require a Kubernetes cluster with Kubeflow Pipelines installed in order to make use of the
 artifact storage provided by [Minio](https://docs.minio.io/) and need to run in the `kubeflow` namespace in order to
 access secrets:
 
     kubectl apply -f <pipeline.yaml> -n kubeflow
     tkn pipeline start <pipeline-name> --showlog -n kubeflow
 
+You can also run the dynamically generated end-to-end test suite which takes all of the "golden" YAML files from the
+compiler `testdata` directory and runs them on a Kubernetes cluster, prerequisite that the environment variable 
+`KUBECONFIG` is set and the K8s cluster has both Kubeflow Pipelines as well as Tekton Pipelines installed:
+
+    make e2e_test
 
 
 ### Compiler Test Report

sdk/python/kfp_tekton/compiler/compiler.py

@@ -336,8 +336,20 @@ def _workflow_with_pipelinerun(self, task_refs, pipeline, pipeline_template, wor
     task_run_spec = []
     for task in task_refs:
       op = pipeline.ops.get(task['name'])
-      task_spec = {"pipelineTaskName": task['name'],
-                   "taskPodTemplate": {}}
+
+      # TODO: should loop-item tasks be included here?
+      if LoopArguments.LOOP_ITEM_NAME_BASE in task['name']:
+        task_name = re.sub(r'-%s-.+$' % LoopArguments.LOOP_ITEM_NAME_BASE, '', task['name'])
+      else:
+        task_name = task['name']
+      op = pipeline.ops.get(task_name)
+      if not op:
+        raise RuntimeError("unable to find op with name '%s'" % task["name"])
+
+      task_spec = {
+        "pipelineTaskName": task['name'],
+        "taskPodTemplate": {}
+      }
       if op.affinity:
         task_spec["taskPodTemplate"]["affinity"] = convert_k8s_obj_to_json(op.affinity)
       if op.tolerations:

sdk/python/tests/compiler/compiler_tests.py

@@ -262,7 +262,7 @@ def test_basic_no_decorator(self):
     from .testdata import basic_no_decorator
     parameter_dict = {
       "function": basic_no_decorator.save_most_frequent_word,
-      "name": 'Save Most Frequent',
+      "name": 'Save Most Frequent Word',
       "description": 'Get Most Frequent Word and Save to GCS',
       "paramsList": [basic_no_decorator.message_param, basic_no_decorator.output_path_param]
     }
@@ -285,7 +285,7 @@ def test_compose(self):
   def _test_pipeline_workflow(self,
                               pipeline_function,
                               pipeline_yaml,
-                              generate_pipelinerun=False,
+                              generate_pipelinerun=True,
                               enable_artifacts=False,
                               normalize_compiler_output_function=None):
     test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
@@ -313,7 +313,9 @@ def _test_workflow_without_decorator(self, pipeline_yaml, params_dict):
     golden_yaml_file = os.path.join(test_data_dir, pipeline_yaml)
     temp_dir = tempfile.mkdtemp()
     try:
-      compiled_workflow = compiler.TektonCompiler()._create_workflow(
+      kfp_tekton_compiler = compiler.TektonCompiler()
+      kfp_tekton_compiler.generate_pipelinerun = True
+      compiled_workflow = kfp_tekton_compiler._create_workflow(
           params_dict['function'],
           params_dict.get('name', None),
           params_dict.get('description', None),
@@ -326,7 +328,7 @@ def _test_workflow_without_decorator(self, pipeline_yaml, params_dict):
   def _test_nested_workflow(self,
                             pipeline_yaml,
                             pipeline_list,
-                            generate_pipelinerun=False,
+                            generate_pipelinerun=True,
                             enable_artifacts=False,
                             normalize_compiler_output_function=None):
     """