[air examples] remove runtime env on dependencies (#46759)

use package from the image Signed-off-by: Lonnie Liu <[email protected]>
ray-project · Jul 25, 2024 · 6b81634 · 6b81634
1 parent 7af3ff0
commit 6b81634
Show file tree

Hide file tree

Showing 7 changed files with 10 additions and 52 deletions.
diff --git a/release/air_examples/dolly_v2_lightning_fsdp_finetuning/dolly_v2_fsdp_env.yaml b/release/air_examples/dolly_v2_lightning_fsdp_finetuning/dolly_v2_fsdp_env.yaml
@@ -3,17 +3,6 @@ env_vars: {}
 debian_packages:
   - curl
 
-python:
-  pip_packages:
-    - "datasets"
-    - "evaluate"
-    - "scikit-learn"
-    - "boto3"
-    - myst-parser==1.0.0
-    - myst-nb==1.0.0
-    - jupytext==1.13.6
-  conda_packages: []
-
 post_build_cmds:
   - pip uninstall -y ray || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }}
   - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }}

diff --git a/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_env.yaml b/release/air_examples/gptj_deepspeed_finetuning/gptj_deepspeed_env.yaml
@@ -3,19 +3,6 @@ env_vars: {}
 debian_packages:
   - curl
 
-python:
-  pip_packages:
-    - "datasets"
-    - "evaluate"
-    - "accelerate==0.16.0"
-    - "transformers==4.26.0"
-    - "torch>=1.12.0"
-    - "deepspeed==0.12.3"
-    - myst-parser==1.0.0
-    - myst-nb==1.0.0
-    - jupytext==1.13.6
-  conda_packages: []
-
 post_build_cmds:
   - pip uninstall -y ray || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }}
-  - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }}
+  - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }}
diff --git a/release/air_examples/opt_deepspeed_batch_inference/30b_deepspeed_env.yaml b/release/air_examples/opt_deepspeed_batch_inference/30b_deepspeed_env.yaml
@@ -3,16 +3,6 @@ env_vars: {}
 debian_packages:
   - curl
 
-python:
-  pip_packages:
-    # Install boto3 to workaround a temporary issue with python and urllib3.
-    # TODO(jungong) : remove this.
-    - boto3
-    - myst-parser==1.0.0
-    - myst-nb==1.0.0
-    - jupytext==1.13.6
-  conda_packages: []
-
 post_build_cmds:
   - pip uninstall -y ray || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }}
   - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }}
diff --git a/release/ray_release/byod/byod_dolly_test.sh b/release/ray_release/byod/byod_dolly_test.sh
@@ -6,4 +6,4 @@ set -exo pipefail
 
 pip3 uninstall -y pytorch-lightning
 pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
-pip3 install "lightning==2.0.2" "transformers==4.29.2" "accelerate==0.19.0"
+pip3 install lightning==2.0.3 myst-parser==1.0.0 myst-nb==1.1.0
diff --git a/release/ray_release/byod/byod_gptj_test.sh b/release/ray_release/byod/byod_gptj_test.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -exo pipefail
+
+pip3 install myst-parser==1.0.0 myst-nb==1.1.0
diff --git a/release/ray_release/byod/byod_vicuna_test.sh b/release/ray_release/byod/byod_vicuna_test.sh
@@ -11,6 +11,7 @@ mkdir -p /mnt/local_storage
 sudo chmod 0777 /mnt/local_storage
 sudo mount /dev/nvme1n1 /mnt/local_storage || true
 EOF
+
 pip3 uninstall -y pytorch-lightning
 pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
-pip3 install lightning==2.0.3  transformers==4.30.2 accelerate==0.20.3 deepspeed==0.12.3
+pip3 install lightning==2.0.3 myst-parser==1.0.0 myst-nb==1.1.0
diff --git a/release/release_tests.yaml b/release/release_tests.yaml
@@ -847,10 +847,7 @@
   cluster:
     byod:
       type: gpu
-      pip:
-        - myst-parser==1.0.0
-        - myst-nb==1.0.0
-        - jupytext==1.13.6
+      post_build_script: byod_gptj_test.sh
     cluster_compute: gptj_deepspeed_compute_aws.yaml
 
   run:
@@ -865,14 +862,6 @@
   cluster:
     byod:
       type: gpu
-      pip:
-        - "datasets"
-        - "evaluate"
-        - "scikit-learn"
-        - "boto3"
-        - myst-parser==1.0.0
-        - myst-nb==1.0.0
-        - jupytext==1.13.6
       post_build_script: byod_dolly_test.sh
     cluster_compute: dolly_v2_fsdp_compute_aws.yaml
 
@@ -890,9 +879,6 @@
   cluster:
     byod:
       type: gpu
-      pip:
-        - myst-parser==1.0.0
-        - myst-nb==1.0.0
       post_build_script: byod_vicuna_test.sh
     cluster_compute: vicuna_13b_deepspeed_compute_aws.yaml