Support artifact outputs

sdk/README.md

@@ -105,7 +105,7 @@ Prerequisite: Install [Kubeflow Pipeline](https://www.kubeflow.org/docs/pipeline
 By default, artifacts are disabled because it's depended on Kubeflow Pipeline's minio setup. When artifacts are enabled, all the output parameters are also treated as artifacts and persist to the default object storage. Enabling artifacts also allow files to be downloaded or stored as artifact inputs/outputs. Since artifacts are depending on the Kubeflow Pipeline's setup by default, the generated Tekton pipeline must be deployed to the same namespace as Kubeflow Pipeline.
 
 To compile Kubeflow Pipelines as Tekton pipelineRun, simply add the `--enable-artifacts` as part of your `dsl-compile-tekton` commands. e.g.
-- `dsl-compile-tekton --py sdk/python/tests/compiler/testdata/artifact_location. --output pipeline.yaml --enable-artifacts`
+- `dsl-compile-tekton --py sdk/python/tests/compiler/testdata/artifact_location.py --output pipeline.yaml --enable-artifacts`
 
 ## Troubleshooting
 - Please be aware that defined Affinity, Node Selector, and Tolerations are applied to all the tasks in the same pipeline because there's only one podTemplate allowed in each pipeline.

sdk/python/tests/compiler/compiler_tests.py

@@ -171,8 +171,8 @@ def test_artifact_location_workflow(self):
     """
     Test compiling a artifact location workflow.
     """
-    from .testdata.artifact_location import foo_pipeline
-    self._test_pipeline_workflow(foo_pipeline, 'artifact_location.yaml', enable_artifacts=True)
+    from .testdata.artifact_location import custom_artifact_location
+    self._test_pipeline_workflow(custom_artifact_location, 'artifact_location.yaml', enable_artifacts=True)
 
   def test_katib_workflow(self):
     """

sdk/python/tests/compiler/testdata/artifact_location.py

@@ -12,41 +12,41 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import kfp
 from kfp import dsl
-from kubernetes.client.models import V1SecretKeySelector
+from kubernetes.client import V1SecretKeySelector
 
 
-@dsl.pipeline(name='artifact-location-pipeine', description='hello world')
-def foo_pipeline(tag: str, namespace: str = "kubeflow", bucket: str = "foobar"):
+@dsl.pipeline(
+    name="custom_artifact_location_pipeline",
+    description="""A pipeline to demonstrate how to configure the artifact
+    location for all the ops in the pipeline.""",
+)
+def custom_artifact_location(
+    secret_name: str = "mlpipeline-minio-artifact",
+    tag: str = '1.31.0',
+    namespace: str = "kubeflow",
+    bucket: str = "mlpipeline"
+):
 
     # configures artifact location
     pipeline_artifact_location = dsl.ArtifactLocation.s3(
-                            bucket=bucket,
-                            endpoint="minio-service.%s:9000" % namespace,
-                            insecure=True,
-                            access_key_secret={"name": "minio", "key": "accesskey"},
-                            secret_key_secret=V1SecretKeySelector(name="minio", key="secretkey"))
-
-    # configures artifact location using AWS IAM role (no access key provided)
-    aws_artifact_location = dsl.ArtifactLocation.s3(
-                            bucket=bucket,
-                            endpoint="s3.amazonaws.com",
-                            region="ap-southeast-1",
-                            insecure=False)
+        bucket=bucket,
+        endpoint="minio-service.%s:9000" % namespace,  # parameterize minio-service endpoint
+        insecure=True,
+        access_key_secret=V1SecretKeySelector(name=secret_name, key="accesskey"),
+        secret_key_secret={"name": secret_name, "key": "secretkey"},  # accepts dict also
+    )
 
     # set pipeline level artifact location
     dsl.get_pipeline_conf().set_artifact_location(pipeline_artifact_location)
 
-    # pipeline level artifact location (to minio)
-    op1 = dsl.ContainerOp(
-        name='foo', 
-        image='busybox:%s' % tag,
-        output_artifact_paths={
-            'out_art': '/tmp/out_art.txt',
-        },
-    )
+    # artifacts in this op are stored to endpoint `minio-service.<namespace>:9000`
+    op = dsl.ContainerOp(name="foo", image="busybox:%s" % tag,
+                         command=['sh', '-c', 'echo hello > /tmp/output.txt'],
+                         file_outputs={'output': '/tmp/output.txt'})
 
 if __name__ == '__main__':
     # don't use top-level import of TektonCompiler to prevent monkey-patching KFP compiler when using KFP's dsl-compile
     from kfp_tekton.compiler import TektonCompiler
-    TektonCompiler().compile(foo_pipeline, __file__.replace('.py', '.yaml'), enable_artifacts=True)
+    TektonCompiler().compile(custom_artifact_location, __file__.replace('.py', '.yaml'), enable_artifacts=True)

sdk/python/tests/compiler/testdata/artifact_location.yaml

@@ -20,13 +20,17 @@ spec:
   params:
   - name: bucket
   - name: namespace
+  - name: secret_name
   - name: tag
-  stepTemplate:
-    volumeMounts:
-    - mountPath: /tmp
-      name: out_art
+  results:
+  - description: /tmp/output.txt
+    name: output
   steps:
-  - image: busybox:$(inputs.params.tag)
+  - command:
+    - sh
+    - -c
+    - echo hello > $(results.output.path)
+    image: busybox:$(inputs.params.tag)
     name: foo
   - env:
     - name: PIPELINERUN
@@ -45,37 +49,40 @@ spec:
       valueFrom:
         secretKeyRef:
           key: accesskey
-          name: minio
+          name: $(inputs.params.secret_name)
     - name: AWS_SECRET_ACCESS_KEY
       valueFrom:
         secretKeyRef:
           key: secretkey
-          name: minio
+          name: $(inputs.params.secret_name)
     image: minio/mc
     name: copy-artifacts
     script: |-
       #!/usr/bin/env sh
       mc config host add storage http://minio-service.$(inputs.params.namespace):9000 $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
-      mc cp /tmp/out_art.txt storage/$(inputs.params.bucket)/runs/$PIPELINERUN/$PODNAME/out_art.txt
-  volumes:
-  - emptyDir: {}
-    name: out_art
+      mc cp $(results.output.path) storage/$(inputs.params.bucket)/runs/$PIPELINERUN/$PODNAME/output.txt
 ---
 apiVersion: tekton.dev/v1beta1
 kind: Pipeline
 metadata:
   annotations:
-    pipelines.kubeflow.org/pipeline_spec: '{"description": "hello world", "inputs":
-      [{"name": "tag", "type": "String"}, {"default": "kubeflow", "name": "namespace",
-      "optional": true, "type": "String"}, {"default": "foobar", "name": "bucket",
-      "optional": true, "type": "String"}], "name": "artifact-location-pipeine"}'
-  name: artifact-location-pipeine
+    pipelines.kubeflow.org/pipeline_spec: '{"description": "A pipeline to demonstrate
+      how to configure the artifact\n    location for all the ops in the pipeline.",
+      "inputs": [{"default": "mlpipeline-minio-artifact", "name": "secret_name", "optional":
+      true, "type": "String"}, {"default": "1.31.0", "name": "tag", "optional": true,
+      "type": "String"}, {"default": "kubeflow", "name": "namespace", "optional":
+      true, "type": "String"}, {"default": "mlpipeline", "name": "bucket", "optional":
+      true, "type": "String"}], "name": "custom_artifact_location_pipeline"}'
+  name: custom-artifact-location-pipeline
 spec:
   params:
-  - name: tag
+  - default: mlpipeline-minio-artifact
+    name: secret_name
+  - default: 1.31.0
+    name: tag
   - default: kubeflow
     name: namespace
-  - default: foobar
+  - default: mlpipeline
     name: bucket
   tasks:
   - name: foo
@@ -84,6 +91,8 @@ spec:
       value: $(params.bucket)
     - name: namespace
       value: $(params.namespace)
+    - name: secret_name
+      value: $(params.secret_name)
     - name: tag
       value: $(params.tag)
     taskRef: