Install Poetry with pipx

GitHub Actions In GitHub Actions, the following steps will be taken to install Poetry and verify that the correct version is installed: 1. Set the `PIPX_VERSION` and `POETRY_VERSION` environment variables, which will be used to install specific versions of each package 2. Install `pipx` with `pip`, for the appropriate version of Python (`pipx` is included by default in the GitHub Actions environment, but only for the default Python version, not necessarily the version installed by actions/setup-python) 3. Install Poetry with `pipx` instead of get-poetry.py/install-poetry.py 4. Test that the output of `poetry -V` matches `$POETRY_VERSION` --- Docker There are conflicting conventions when working with Poetry in Docker: 1. Docker's convention is to not use virtualenvs, because containers themselves provide sufficient isolation. 2. Poetry's convention is to always use virtualenvs. This project has previously preferred the Docker convention: - Poetry itself was installed with the get-poetry.py script, with the environment variable `POETRY_HOME=/opt/poetry` used to specify a consistent location for Poetry. - `poetry install` was used with `POETRY_VIRTUALENVS_CREATE=false` to install the project's packages into the system Python directory. The conventional Docker approach no longer works because: - The old install script get-poetry.py is deprecated and not compatible with Python 3.10. - The new install script install-poetry.py has been problematic so far, and Poetry doesn't really test it, so problems will likely continue. In the updated approach: - `ENV PATH=/opt/pipx/bin:/app/.venv/bin:$PATH` will prepare `$PATH`. - `pip` will be used to install a pinned version of `pipx`. - `pipx` will be used to install a pinned version of Poetry, with `PIPX_BIN_DIR=/opt/pipx/bin` used to specify the location where `pipx` installs the Poetry command-line application, and `PIPX_HOME=/opt/pipx/home` used as the location for `pipx` itself. - `poetry install` will be used with `POETRY_VIRTUALENVS_CREATE=true`, `POETRY_VIRTUALENVS_IN_PROJECT=true` and `WORKDIR /app` to install the project's packages into a virtualenv at `/app/.venv`. Subsequent `python` commands will use `app/.venv/bin/python`. As long as `POETRY_VIRTUALENVS_IN_PROJECT=true` and `WORKDIR /app` are retained, subsequent Poetry commands will use the same virtualenv at `/app/.venv`.
br3ndonland · Oct 26, 2021 · af7bedd · af7bedd
1 parent 48ae350
commit af7bedd
Show file tree

Hide file tree

Showing 5 changed files with 139 additions and 34 deletions.
diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml
@@ -9,9 +9,9 @@ on:
   workflow_dispatch:
 
 env:
-  POETRY_HOME: /opt/poetry
-  POETRY_VERSION: 1.1.7
-  POETRY_VIRTUALENVS_CREATE: false
+  PIPX_VERSION: "0.16.4"
+  POETRY_VERSION: "1.1.11"
+  POETRY_VIRTUALENVS_IN_PROJECT: true
 
 jobs:
   python:
@@ -38,12 +38,10 @@ jobs:
           path: ~/.cache/pre-commit
           key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
           restore-keys: ${{ runner.os }}-pre-commit-
+      - name: Install pipx for Python ${{ matrix.python-version }}
+        run: python -m pip install "pipx==$PIPX_VERSION"
       - name: Install Poetry
-        run: |
-          curl -fsS -o get-poetry.py \
-            "https://raw.githubusercontent.com/python-poetry/poetry/$POETRY_VERSION/get-poetry.py"
-          python get-poetry.py -y
-          echo "$POETRY_HOME/bin" >> $GITHUB_PATH
+        run: pipx install "poetry==$POETRY_VERSION"
       - name: Test Poetry version
         run: |
           POETRY_VERSION_INSTALLED=$(poetry -V)
@@ -55,10 +53,20 @@ jobs:
           esac
       - name: Install dependencies
         run: poetry install --no-interaction -E fastapi
+      - name: Test virtualenv location
+        run: |
+          EXPECTED_VIRTUALENV_PATH=${{ github.workspace }}/.venv
+          INSTALLED_VIRTUALENV_PATH=$(poetry env info --path)
+          echo "The virtualenv should be at $EXPECTED_VIRTUALENV_PATH."
+          echo "Poetry is using a virtualenv at $INSTALLED_VIRTUALENV_PATH."
+          case "$INSTALLED_VIRTUALENV_PATH" in
+          "$EXPECTED_VIRTUALENV_PATH") echo "Correct Poetry virtualenv." ;;
+          *) echo "Incorrect Poetry virtualenv." && exit 1 ;;
+          esac
       - name: Run pre-commit hooks
-        run: pre-commit run --all-files
+        run: poetry run pre-commit run --all-files
       - name: Run unit tests
-        run: pytest --cov-report=xml
+        run: poetry run pytest --cov-report=xml
       - name: Upload test coverage report to Codecov
         uses: codecov/codecov-action@v2
         with:
@@ -71,6 +79,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: [python]
     strategy:
+      fail-fast: false
       matrix:
         linux-version: ["", "alpine", "slim"]
         python-version: [3.8, 3.9]
@@ -95,17 +104,20 @@ jobs:
         run: |
           docker build . --rm --target base \
             --build-arg LINUX_VERSION="$LINUX_VERSION" \
+            --build-arg PIPX_VERSION="$PIPX_VERSION" \
             --build-arg POETRY_VERSION="$POETRY_VERSION" \
             --build-arg PYTHON_VERSION="$PYTHON_VERSION" \
             --cache-from python:"$PYTHON_VERSION$LINUX_TAG" \
             -t ghcr.io/br3ndonland/inboard:base"$LINUX_TAG"
           docker build . --rm --target starlette \
             --build-arg LINUX_VERSION="$LINUX_VERSION" \
+            --build-arg PIPX_VERSION="$PIPX_VERSION" \
             --build-arg POETRY_VERSION="$POETRY_VERSION" \
             --build-arg PYTHON_VERSION="$PYTHON_VERSION" \
             -t ghcr.io/br3ndonland/inboard:starlette"$LINUX_TAG"
           docker build . --rm --target fastapi \
             --build-arg LINUX_VERSION="$LINUX_VERSION" \
+            --build-arg PIPX_VERSION="$PIPX_VERSION" \
             --build-arg POETRY_VERSION="$POETRY_VERSION" \
             --build-arg PYTHON_VERSION="$PYTHON_VERSION" \
             -t ghcr.io/br3ndonland/inboard:fastapi"$LINUX_TAG"
@@ -140,6 +152,28 @@ jobs:
             done
           }
           test_poetry_version_in_docker inboard-base inboard-starlette inboard-fastapi
+      - name: Test virtualenv location in Docker containers
+        run: |
+          test_virtualenv_location_in_docker() {
+            local docker_poetry docker_python expected_poetry expected_python
+            expected_poetry="/app/.venv"
+            expected_python="$expected_poetry/bin/python"
+            echo "The Poetry virtualenv should be at $expected_poetry."
+            echo "The Python executable should be at $expected_python."
+            for container_name in "$@"; do
+              docker_poetry=$(docker exec "$container_name" poetry env info --path)
+              docker_python=$(docker exec "$container_name" which python)
+              case "$docker_poetry" in
+              "$expected_poetry") echo "Correct Poetry virtualenv $docker_poetry for $container_name." ;;
+              *) echo "Incorrect Poetry virtualenv $docker_poetry for $container_name." && return 1 ;;
+              esac
+              case "$docker_python" in
+              "$expected_python") echo "Correct Python $docker_python for $container_name." ;;
+              *) echo "Incorrect Python $docker_python for $container_name." && return 1 ;;
+              esac
+            done
+          }
+          test_virtualenv_location_in_docker inboard-base inboard-starlette inboard-fastapi
       - name: Smoke test Docker containers
         run: |
           handle_error_code() {

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -1,17 +1,71 @@
 name: codeql
 
 on:
+  push:
+    branches: [develop, main]
   pull_request:
+    branches: [develop, main]
   schedule:
     - cron: "0 13 * * 1"
   workflow_dispatch:
 
+env:
+  PIPX_VERSION: "0.16.4"
+  POETRY_VERSION: "1.1.11"
+  POETRY_VIRTUALENVS_IN_PROJECT: true
+
 jobs:
   analyze:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9]
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
     steps:
       - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Set up Poetry cache for Python dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: ${{ runner.os }}-poetry-
+      - name: Install pipx for Python ${{ matrix.python-version }}
+        run: python -m pip install "pipx==$PIPX_VERSION"
+      - name: Install Poetry
+        run: pipx install "poetry==$POETRY_VERSION"
+      - name: Test Poetry version
+        run: |
+          POETRY_VERSION_INSTALLED=$(poetry -V)
+          echo "The POETRY_VERSION environment variable is set to $POETRY_VERSION."
+          echo "The installed Poetry version is $POETRY_VERSION_INSTALLED."
+          case $POETRY_VERSION_INSTALLED in
+          *$POETRY_VERSION*) echo "Poetry version correct." ;;
+          *) echo "Poetry version incorrect." && exit 1 ;;
+          esac
+      - name: Install dependencies
+        run: poetry install --no-interaction -E fastapi
+      - name: Test Poetry virtualenv location
+        run: |
+          EXPECTED_VIRTUALENV_PATH=${{ github.workspace }}/.venv
+          INSTALLED_VIRTUALENV_PATH=$(poetry env info --path)
+          echo "The virtualenv should be at $EXPECTED_VIRTUALENV_PATH."
+          echo "Poetry is using a virtualenv at $INSTALLED_VIRTUALENV_PATH."
+          case "$INSTALLED_VIRTUALENV_PATH" in
+          "$EXPECTED_VIRTUALENV_PATH") echo "Correct Poetry virtualenv." ;;
+          *) echo "Incorrect Poetry virtualenv." && exit 1 ;;
+          esac
+          echo "INSTALLED_VIRTUALENV_PATH=$INSTALLED_VIRTUALENV_PATH" >> $GITHUB_ENV
+      - name: Set the `CODEQL_PYTHON` environment variable so CodeQL can find dependencies
+        run: echo "CODEQL_PYTHON=$INSTALLED_VIRTUALENV_PATH/bin/python" >> $GITHUB_ENV
       - uses: github/codeql-action/init@v1
         with:
           languages: python
+          setup-python-dependencies: false
+      - uses: github/codeql-action/autobuild@v1
       - uses: github/codeql-action/analyze@v1
diff --git a/.github/workflows/hooks.yml b/.github/workflows/hooks.yml
@@ -7,9 +7,9 @@ on:
   workflow_dispatch:
 
 env:
-  POETRY_HOME: /opt/poetry
-  POETRY_VERSION: 1.1.7
-  POETRY_VIRTUALENVS_CREATE: false
+  PIPX_VERSION: "0.16.4"
+  POETRY_VERSION: "1.1.11"
+  POETRY_VIRTUALENVS_IN_PROJECT: true
 
 jobs:
   pre-commit:
@@ -36,12 +36,10 @@ jobs:
           path: ~/.cache/pre-commit
           key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
           restore-keys: ${{ runner.os }}-pre-commit-
+      - name: Install pipx for Python ${{ matrix.python-version }}
+        run: python -m pip install "pipx==$PIPX_VERSION"
       - name: Install Poetry
-        run: |
-          curl -fsS -o get-poetry.py \
-            "https://raw.githubusercontent.com/python-poetry/poetry/$POETRY_VERSION/get-poetry.py"
-          python get-poetry.py -y
-          echo "$POETRY_HOME/bin" >> $GITHUB_PATH
+        run: pipx install "poetry==$POETRY_VERSION"
       - name: Test Poetry version
         run: |
           POETRY_VERSION_INSTALLED=$(poetry -V)
@@ -53,5 +51,15 @@ jobs:
           esac
       - name: Install dependencies
         run: poetry install --no-interaction -E fastapi
+      - name: Test virtualenv location
+        run: |
+          EXPECTED_VIRTUALENV_PATH=${{ github.workspace }}/.venv
+          INSTALLED_VIRTUALENV_PATH=$(poetry env info --path)
+          echo "The virtualenv should be at $EXPECTED_VIRTUALENV_PATH."
+          echo "Poetry is using a virtualenv at $INSTALLED_VIRTUALENV_PATH."
+          case "$INSTALLED_VIRTUALENV_PATH" in
+          "$EXPECTED_VIRTUALENV_PATH") echo "Correct Poetry virtualenv." ;;
+          *) echo "Incorrect Poetry virtualenv." && exit 1 ;;
+          esac
       - name: Run pre-commit hooks
-        run: pre-commit run --all-files
+        run: poetry run pre-commit run --all-files
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -11,9 +11,9 @@ on:
   workflow_dispatch:
 
 env:
-  POETRY_HOME: /opt/poetry
-  POETRY_VERSION: 1.1.7
-  POETRY_VIRTUALENVS_CREATE: false
+  PIPX_VERSION: "0.16.4"
+  POETRY_VERSION: "1.1.11"
+  POETRY_VIRTUALENVS_IN_PROJECT: true
 
 jobs:
   test:
@@ -33,12 +33,10 @@ jobs:
           path: ~/.cache/pypoetry
           key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
           restore-keys: ${{ runner.os }}-poetry-
+      - name: Install pipx for Python ${{ matrix.python-version }}
+        run: python -m pip install "pipx==$PIPX_VERSION"
       - name: Install Poetry
-        run: |
-          curl -fsS -o get-poetry.py \
-            "https://raw.githubusercontent.com/python-poetry/poetry/$POETRY_VERSION/get-poetry.py"
-          python get-poetry.py -y
-          echo "$POETRY_HOME/bin" >> $GITHUB_PATH
+        run: pipx install "poetry==$POETRY_VERSION"
       - name: Test Poetry version
         run: |
           POETRY_VERSION_INSTALLED=$(poetry -V)
@@ -50,8 +48,20 @@ jobs:
           esac
       - name: Install dependencies
         run: poetry install --no-interaction -E fastapi
+      - name: Test virtualenv location
+        run: |
+          EXPECTED_VIRTUALENV_PATH=${{ github.workspace }}/.venv
+          INSTALLED_VIRTUALENV_PATH=$(poetry env info --path)
+          echo "The virtualenv should be at $EXPECTED_VIRTUALENV_PATH."
+          echo "Poetry is using a virtualenv at $INSTALLED_VIRTUALENV_PATH."
+          case "$INSTALLED_VIRTUALENV_PATH" in
+          "$EXPECTED_VIRTUALENV_PATH") echo "Correct Poetry virtualenv." ;;
+          *) echo "Incorrect Poetry virtualenv." && exit 1 ;;
+          esac
+      - name: Run pre-commit hooks
+        run: poetry run pre-commit run --all-files
       - name: Run unit tests
-        run: pytest --cov-report=xml
+        run: poetry run pytest --cov-report=xml
       - name: Upload test coverage report to Codecov
         uses: codecov/codecov-action@v2
         with:

diff --git a/Dockerfile b/Dockerfile
@@ -6,14 +6,13 @@ LABEL org.opencontainers.image.licenses="MIT"
 LABEL org.opencontainers.image.source="https://github.com/br3ndonland/inboard"
 LABEL org.opencontainers.image.title="inboard"
 LABEL org.opencontainers.image.url="https://github.com/br3ndonland/inboard/pkgs/container/inboard"
-ARG LINUX_VERSION POETRY_VERSION=1.1.7
-ENV APP_MODULE=inboard.app.main_base:app LINUX_VERSION=$LINUX_VERSION PATH=/opt/poetry/bin:$PATH POETRY_HOME=/opt/poetry POETRY_VERSION=$POETRY_VERSION POETRY_VIRTUALENVS_CREATE=false PYTHONPATH=/app
+ARG LINUX_VERSION PIPX_VERSION=0.16.4 POETRY_VERSION=1.1.11
+ENV APP_MODULE=inboard.app.main_base:app LINUX_VERSION=$LINUX_VERSION PATH=/opt/pipx/bin:/app/.venv/bin:$PATH PIPX_BIN_DIR=/opt/pipx/bin PIPX_HOME=/opt/pipx/home PIPX_VERSION=$PIPX_VERSION POETRY_VERSION=$POETRY_VERSION POETRY_VIRTUALENVS_IN_PROJECT=true PYTHONPATH=/app
 COPY poetry.lock pyproject.toml /app/
-WORKDIR /app/
+WORKDIR /app
 RUN sh -c 'if [ "$LINUX_VERSION" = "slim" ]; then apt-get update -qy && apt-get install -qy --no-install-recommends gcc libc-dev make wget; fi' && \
-  wget -qO get-poetry.py "https://raw.githubusercontent.com/python-poetry/poetry/$POETRY_VERSION/get-poetry.py" && \
-  sh -c '. /etc/os-release; if [ "$ID" = "alpine" ]; then apk add --no-cache --virtual .build-deps gcc libc-dev make; fi' && \
-  python get-poetry.py -y && poetry install --no-dev --no-interaction --no-root && \
+  sh -c '. /etc/os-release; if [ "$ID" = "alpine" ]; then apk add --no-cache --virtual .build-deps gcc libc-dev libffi-dev make openssl-dev; fi' && \
+  python -m pip install --no-cache-dir --upgrade pip "pipx==$PIPX_VERSION" && pipx install "poetry==$POETRY_VERSION" && poetry install --no-dev --no-interaction --no-root && \
   sh -c 'if [ "$LINUX_VERSION" = "slim" ]; then apt-get purge --auto-remove -qy gcc libc-dev make wget; fi' && \
   sh -c '. /etc/os-release; if [ "$ID" = "alpine" ]; then apk del .build-deps; fi'
 COPY inboard /app/inboard