Replace boto3 with numpy

.github/scripts/python_package_upload/Dockerfile

@@ -0,0 +1,15 @@
+FROM docker.io/python:3.9
+
+# Set working directory
+WORKDIR /app
+
+# Copy the script into the container
+COPY package_upload.sh /app/package_upload.sh
+
+# Make sure the script is executable
+RUN chmod +x /app/package_upload.sh
+
+# Store the files in a folder
+VOLUME /app/packages
+
+ENTRYPOINT ["/app/package_upload.sh"]

.github/scripts/python_package_upload/package_upload.sh

@@ -1,33 +1,12 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 set -ex
 
-kfp_directory=kfp
-boto_directory=boto3
-
-mkdir -p "$kfp_directory"
-mkdir -p "$boto_directory"
-
-# Download kfp package
-pip download kfp==2.7.0 -d "$kfp_directory"
-
-# Download boto3 package
-pip download boto3 -d "$boto_directory"
-
-
-# Print the pods in the namespace
-oc -n test-pypiserver get pods
-
-pod_name=$(oc -n test-pypiserver get pod | grep pypi | awk '{print $1}')
-
-# Copy kfp packages
-for kfp_entry in "$kfp_directory"/*; do
-    echo oc -n test-pypiserver cp "$kfp_entry" $pod_name:/opt/app-root/packages
-    oc -n test-pypiserver cp "$kfp_entry" $pod_name:/opt/app-root/packages
-done
-
-# Copy boto3 packages
-for boto_entry in "$boto_directory"/*; do
-    echo oc -n test-pypiserver cp "$boto_entry" $pod_name:/opt/app-root/packages
-    oc -n test-pypiserver cp "$boto_entry" $pod_name:/opt/app-root/packages
+# Download packages
+for python_version in "3.9" "3.10" "3.11" "3.12"; do
+  for package in "kfp" "numpy"; do
+    # If we don't set the --python it will use the one from the computer that may not be the one that the
+    # pipeline is running
+    pip download $package -d packages --only-binary=:none: --python $python_version
+  done
 done

.github/scripts/python_package_upload/package_upload_run.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -ex
+
+mkdir -p /tmp/packages
+docker rm package_upload_run || true
+docker build -t package_upload .
+docker run --name package_upload_run -v /tmp/packages:/app/packages package_upload
+
+# Print the pods in the namespace
+oc -n test-pypiserver get pods
+
+pod_name=$(oc -n test-pypiserver get pod | grep pypi | awk '{print $1}')
+
+# Copy packages
+for entry in /tmp/packages/*; do
+    oc -n test-pypiserver cp "$entry" $pod_name:/opt/app-root/packages
+done

.github/scripts/tests/tests.sh

@@ -128,7 +128,7 @@ upload_python_packages_to_pypi_server() {
   echo "---------------------------------"
   echo "Upload Python Packages to pypi-server"
   echo "---------------------------------"
-  ( cd "${GIT_WORKSPACE}/.github/scripts/python_package_upload" && sh package_upload.sh )
+  ( cd "${GIT_WORKSPACE}/.github/scripts/python_package_upload" && sh package_upload_run.sh )
 }
 
 create_dspa_namespace() {

.gitignore

@@ -34,4 +34,3 @@ Dockerfile.cross
 __pycache__/
 *.py[cod]
 *$py.class
-.github/scripts/python_package_upload/

tests/resources/test-pipeline-run.yaml

@@ -20,29 +20,29 @@ deploymentSpec:
     exec-say-hello:
       container:
         args:
-          - --executor_input
-          - '{{$}}'
-          - --function_to_execute
-          - say_hello
+        - --executor_input
+        - '{{$}}'
+        - --function_to_execute
+        - say_hello
         command:
-          - sh
-          - -c
-          - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
+        - sh
+        - -c
+        - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
           \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
-          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.3.0'\
+          \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.9.0'\
           \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\
           $0\" \"$@\"\n"
-          - sh
-          - -ec
-          - 'program_path=$(mktemp -d)
+        - sh
+        - -ec
+        - 'program_path=$(mktemp -d)
 
 
           printf "%s" "$0" > "$program_path/ephemeral_component.py"
 
           _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main                         --component_module_path                         "$program_path/ephemeral_component.py"                         "$@"
 
           '
-          - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
+        - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
           \ *\n\ndef say_hello(name: str) -> str:\n    hello_text = f'Hello, {name}!'\n\
           \    print(hello_text)\n    return hello_text\n\n"
         image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.1
@@ -77,4 +77,4 @@ root:
       Output:
         parameterType: STRING
 schemaVersion: 2.1.0
-sdkVersion: kfp-2.3.0
+sdkVersion: kfp-2.9.0

tests/resources/test-pipeline-with-custom-pip-server-run.yaml

@@ -22,13 +22,14 @@ deploymentSpec:
         - sh
         - -c
         - "\nif ! [ -x \"$(command -v pip)\" ]; then\n    python3 -m ensurepip ||\
-          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n
-          \ PIP_DISABLE_PIP_VERSION_CHECK=1\
-          \ python3 -m pip install --quiet --no-warn-script-location --index-url  https://nginx-service.test-pypiserver.svc.cluster.local:443/simple/\
-          \ 'kfp==2.7.0' '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"\
-          3.9\"'  &&  python3 -m pip install --quiet --no-warn-script-location --index-url\
-          \  https://nginx-service.test-pypiserver.svc.cluster.local:443/simple/ 'boto3' && \"\
-          $0\" \"$@\"\n"
+          \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
+          \ python3 -m pip install --quiet --no-warn-script-location --index-url https://nginx-service.test-pypiserver.svc.cluster.local/simple/\
+          \ --trusted-host nginx-service.test-pypiserver.svc.cluster.local 'kfp==2.9.0'\
+          \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"'  &&\
+          \  python3 -m pip install --quiet --no-warn-script-location --index-url\
+          \ https://nginx-service.test-pypiserver.svc.cluster.local/simple/ --trusted-host\
+          \ nginx-service.test-pypiserver.svc.cluster.local 'numpy' && \"$0\" \"$@\"\
+          \n"
         - sh
         - -ec
         - 'program_path=$(mktemp -d)
@@ -40,8 +41,9 @@ deploymentSpec:
 
           '
         - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
-          \ *\n\ndef say_hello() -> str:\n    hello_text = f'Hello!'\n    print(hello_text)\n\
-          \    return hello_text\n\n"
+          \ *\n\ndef say_hello() -> str:\n    import numpy as np\n    hello_text =\
+          \ f'Numpy version: {np.__version__}'\n    print(hello_text)\n    return\
+          \ hello_text\n\n"
         image: quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0
 pipelineInfo:
   name: hello-pipeline
@@ -66,4 +68,4 @@ root:
       Output:
         parameterType: STRING
 schemaVersion: 2.1.0
-sdkVersion: kfp-2.7.0
+sdkVersion: kfp-2.9.0

tests/resources/test-pipeline-with-custom-pip-server.py

@@ -3,10 +3,12 @@
 # Edited the compiled version manually, to remove the --trusted-host flag
 # this is so we can test for tls certs validations when launcher installs packages
 @dsl.component(base_image="quay.io/opendatahub/ds-pipelines-ci-executor-image:v1.0",
-packages_to_install=['boto3'],
-pip_index_urls=['https://nginx-service.test-pypiserver.svc.cluster.local:443/simple/'])
+packages_to_install=['numpy'],
+pip_index_urls=['https://nginx-service.test-pypiserver.svc.cluster.local/simple/'],
+pip_trusted_hosts=['nginx-service.test-pypiserver.svc.cluster.local'])
 def say_hello() -> str:
-    hello_text = 'Hello!'
+    import numpy as np
+    hello_text = f'Numpy version: {np.__version__}'
     print(hello_text)
     return hello_text
 
@@ -18,4 +20,4 @@ def hello_pipeline() -> str:
 
 
 if __name__ == '__main__':
-    compiler.Compiler().compile(hello_pipeline, __file__ + '.yaml')
+    compiler.Compiler().compile(hello_pipeline, __file__.replace('.py', '-run.yaml'))

tests/resources/test-pipeline-src.py → tests/resources/test-pipeline.py

@@ -15,4 +15,4 @@ def hello_pipeline(recipient: str) -> str:
 
 
 if __name__ == '__main__':
-    compiler.Compiler().compile(hello_pipeline, __file__ + '.yaml')
+    compiler.Compiler().compile(hello_pipeline, __file__.replace('.py', '-run.yaml'))