diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8eb870..5345ef5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - image: [all-python] + image: [all-python-cpu, all-python-cuda] env: FORCE_COLOR: 1 steps: diff --git a/Earthfile b/Earthfile index 9a43170..3840f9c 100644 --- a/Earthfile +++ b/Earthfile @@ -11,26 +11,48 @@ ARG --required --global VERSION common: FROM mambaorg/micromamba:1.5.8 ENV SHELL=/bin/bash - RUN micromamba install -n base -c conda-forge -y htop nvtop + RUN micromamba install -n base -c conda-forge -y htop nvtop WORKDIR /workspace all: BUILD +all-python all-python: - BUILD +python-jupyter - BUILD +python-code + BUILD all-python-cpu + BUILD all-python-cuda -python: +all-python-cpu: + BUILD +python-cpu-jupyter + BUILD +python-cpu-code + +all-python-cuda: + BUILD +python-cuda-jupyter + BUILD +python-cuda-code + +python-cpu: FROM +common - DO python+SETUP + DO python+SETUP_CPU -python-jupyter: - FROM +python +python-cpu-jupyter: + FROM +python-cpu DO jupyter+SETUP SAVE IMAGE --push $REGISTRY/python-jupyter:$VERSION -python-code: - FROM +python +python-cpu-code: + FROM +python-cpu DO code+SETUP SAVE IMAGE --push $REGISTRY/python-code:$VERSION + +python-cuda: + FROM +common + DO python+SETUP_CUDA + +python-cuda-jupyter: + FROM +python-cuda + DO jupyter+SETUP + SAVE IMAGE --push $REGISTRY/python-cuda-jupyter:$VERSION + +python-cuda-code: + FROM +python-cuda + DO code+SETUP + SAVE IMAGE --push $REGISTRY/python-cuda-code:$VERSION diff --git a/frontend/code/Earthfile b/frontend/code/Earthfile index 0623781..e232acb 100644 --- a/frontend/code/Earthfile +++ b/frontend/code/Earthfile @@ -2,10 +2,11 @@ VERSION 0.8 SETUP: FUNCTION - RUN micromamba install -n base -c conda-forge -y code-server ipykernel ipywidgets + RUN micromamba install -n base -c conda-forge -y code-server jupyterhub-singleuser jupyter-server-proxy ipykernel ipywidgets RUN micromamba -n base run code-server \ --install-extension ms-toolsai.jupyter \ --install-extension equinusocio.vsc-material-theme \ --install-extension equinusocio.vsc-material-theme-icons \ --install-extension ms-python.python - CMD ["code-server", "--bind-addr=0.0.0.0:8888", "."] + COPY frontend/code/jupyter_server_config.py /etc/jupyter/jupyter_server_config.py + CMD ["jupyter", "server", "--ip=0.0.0.0", "--port=8888"] diff --git a/frontend/code/jupyter_server_config.py b/frontend/code/jupyter_server_config.py new file mode 100644 index 0000000..e8560a0 --- /dev/null +++ b/frontend/code/jupyter_server_config.py @@ -0,0 +1,7 @@ +c = get_config() + +c.ServerProxy.servers = { + "code": { + "command": ["code-server", "--bind-addr=0.0.0.0:{port}", "--auth=none", "."] + } +} diff --git a/language/python/Earthfile b/language/python/Earthfile index 4d39a20..456fe35 100644 --- a/language/python/Earthfile +++ b/language/python/Earthfile @@ -2,11 +2,21 @@ VERSION 0.8 SETUP: FUNCTION - RUN micromamba install -n base -c pytorch -c nvidia -c conda-forge -y \ - python pytorch torchvision torchaudio pytorch-cuda=11.8 \ + ARG COMPUTE_SPECIFIC_CHANNELS + ARG COMPUTE_SPECIFIC_PACKAGES + RUN micromamba install -n base -c pytorch -c conda-forge $COMPUTE_SPECIFIC_CHANNELS -y \ + python pytorch torchvision torchaudio $COMPUTE_SPECIFIC_PACKAGES \ lightning torchinfo tensorboard tensorboardx \ transformers diffusers datasets accelerate evaluate optimum \ numpy pillow \ pandas polars duckdb python-duckdb \ matplotlib seaborn plotly \ gradio streamlit + +SETUP_CPU: + FUNCTION + DO +SETUP --COMPUTE_SPECIFIC_PACKAGES="cpuonly" + +SETUP_CUDA: + FUNCTION + DO +SETUP --COMPUTE_SPECIFIC_CHANNELS="-c nvidia" --COMPUTE_SPECIFIC_PACKAGES="pytorch-cuda=12.1"