From 157ac584c5aafc929f0de71f15086ebdc1d2abc7 Mon Sep 17 00:00:00 2001 From: ydshieh Date: Tue, 27 Sep 2022 15:11:05 +0200 Subject: [PATCH] Build docker images for tokenizers main --- .github/workflows/build-docker-images.yml | 92 +------------------ docker/transformers-all-latest-gpu/Dockerfile | 11 ++- .../Dockerfile | 11 ++- docker/transformers-pytorch-gpu/Dockerfile | 11 ++- docker/transformers-tensorflow-gpu/Dockerfile | 11 ++- 5 files changed, 45 insertions(+), 91 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 88934cbcbc5bb4..3ecc4a911f69eb 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -4,6 +4,7 @@ on: push: branches: - docker-image* + - test-tokenizers-* repository_dispatch: workflow_call: inputs: @@ -42,36 +43,7 @@ jobs: build-args: | REF=main push: true - tags: huggingface/transformers-all-latest-gpu${{ inputs.image_postfix }} - - latest-with-torch-nightly-docker: - name: "Nightly PyTorch + Stable TensorFlow" - # Push CI doesn't need this image - if: inputs.image_postfix != '-push-ci' - runs-on: ubuntu-latest - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Check out code - uses: actions/checkout@v2 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: ./docker/transformers-all-latest-gpu - build-args: | - REF=main - PYTORCH=pre - push: true - tags: huggingface/transformers-all-latest-torch-nightly-gpu + tags: huggingface/internal-transformers-all-latest-gpu-tokenizers-main latest-torch-deepspeed-docker: name: "Latest PyTorch + DeepSpeed" @@ -97,61 +69,7 @@ jobs: build-args: | REF=main push: true - tags: huggingface/transformers-pytorch-deepspeed-latest-gpu${{ inputs.image_postfix }} - - nightly-torch-deepspeed-docker: - name: "Nightly PyTorch + DeepSpeed" - # Push CI doesn't need this image - if: inputs.image_postfix != '-push-ci' - runs-on: ubuntu-latest - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Check out code - uses: actions/checkout@v2 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: ./docker/transformers-pytorch-deepspeed-nightly-gpu - build-args: | - REF=main - push: true - tags: huggingface/transformers-pytorch-deepspeed-nightly-gpu - - doc-builder: - name: "Doc builder" - # Push CI doesn't need this image - if: inputs.image_postfix != '-push-ci' - runs-on: ubuntu-latest - steps: - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Check out code - uses: actions/checkout@v2 - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: ./docker/transformers-doc-builder - push: true - tags: huggingface/transformers-doc-builder + tags: huggingface/internal-transformers-pytorch-deepspeed-latest-gpu-tokenizers-main latest-pytorch: name: "Latest PyTorch [dev]" @@ -179,7 +97,7 @@ jobs: build-args: | REF=main push: true - tags: huggingface/transformers-pytorch-gpu + tags: huggingface/internal-transformers-pytorch-gpu-tokenizers-main latest-tensorflow: name: "Latest TensorFlow [dev]" @@ -207,4 +125,4 @@ jobs: build-args: | REF=main push: true - tags: huggingface/transformers-tensorflow-gpu + tags: huggingface/internal-transformers-tensorflow-gpu-tokenizers-main diff --git a/docker/transformers-all-latest-gpu/Dockerfile b/docker/transformers-all-latest-gpu/Dockerfile index 4db6f51826f02b..2513a4d65075a2 100644 --- a/docker/transformers-all-latest-gpu/Dockerfile +++ b/docker/transformers-all-latest-gpu/Dockerfile @@ -16,7 +16,7 @@ ARG INTEL_TORCH_EXT='1.11.0' ARG CUDA='cu113' RUN apt update -RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg git-lfs +RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg git-lfs curl libssl-dev RUN git lfs install RUN python3 -m pip install --no-cache-dir --upgrade pip @@ -53,3 +53,12 @@ RUN python3 -m pip install --no-cache-dir decord # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. RUN cd transformers && python3 setup.py develop + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN cargo --help +RUN python3 -m pip uninstall -y tokenizers +RUN git clone https://github.com/huggingface/tokenizers +RUN python3 -m pip install --no-cache-dir setuptools-rust +RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python diff --git a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile index f5e29175b41d3f..2c597703e3abc7 100644 --- a/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile +++ b/docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile @@ -8,7 +8,7 @@ ARG PYTORCH='1.12.1' ARG CUDA='cu113' RUN apt -y update -RUN apt install -y libaio-dev +RUN apt install -y libaio-dev curl libssl-dev RUN python3 -m pip install --no-cache-dir --upgrade pip ARG REF=main @@ -33,3 +33,12 @@ RUN DS_BUILD_CPU_ADAM=1 DS_BUILD_FUSED_ADAM=1 DS_BUILD_AIO=1 DS_BUILD_UTILS=1 py RUN cd transformers && python3 setup.py develop RUN python3 -c "from deepspeed.launcher.runner import main" + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN cargo --help +RUN python3 -m pip uninstall -y tokenizers +RUN git clone https://github.com/huggingface/tokenizers +RUN python3 -m pip install --no-cache-dir setuptools-rust +RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python diff --git a/docker/transformers-pytorch-gpu/Dockerfile b/docker/transformers-pytorch-gpu/Dockerfile index 668bec3e715d86..194bf0e333d2ce 100644 --- a/docker/transformers-pytorch-gpu/Dockerfile +++ b/docker/transformers-pytorch-gpu/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="Hugging Face" ARG DEBIAN_FRONTEND=noninteractive RUN apt update -RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg +RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg curl libssl-dev RUN python3 -m pip install --no-cache-dir --upgrade pip ARG REF=main @@ -29,3 +29,12 @@ RUN python3 -m pip install -U "itsdangerous<2.1.0" # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. RUN cd transformers && python3 setup.py develop + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN cargo --help +RUN python3 -m pip uninstall -y tokenizers +RUN git clone https://github.com/huggingface/tokenizers +RUN python3 -m pip install --no-cache-dir setuptools-rust +RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python diff --git a/docker/transformers-tensorflow-gpu/Dockerfile b/docker/transformers-tensorflow-gpu/Dockerfile index a05ace7d08e268..1c5b52aa42f92d 100644 --- a/docker/transformers-tensorflow-gpu/Dockerfile +++ b/docker/transformers-tensorflow-gpu/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="Hugging Face" ARG DEBIAN_FRONTEND=noninteractive RUN apt update -RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg +RUN apt install -y git libsndfile1-dev tesseract-ocr espeak-ng python3 python3-pip ffmpeg curl libssl-dev RUN python3 -m pip install --no-cache-dir --upgrade pip ARG REF=main @@ -21,3 +21,12 @@ RUN python3 -m pip install -U "itsdangerous<2.1.0" # When installing in editable mode, `transformers` is not recognized as a package. # this line must be added in order for python to be aware of transformers. RUN cd transformers && python3 setup.py develop + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN cargo --help +RUN python3 -m pip uninstall -y tokenizers +RUN git clone https://github.com/huggingface/tokenizers +RUN python3 -m pip install --no-cache-dir setuptools-rust +RUN python3 -m pip install --no-cache-dir ./tokenizers/bindings/python