From 892218050c271baddd8b0bac453018ae1da0bc43 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Thu, 31 Oct 2024 16:21:43 +0000 Subject: [PATCH] Misc package manager handling refactor Split out of the Poetry PR to keep that one smaller. --- bin/compile | 8 ++++---- bin/utils | 12 ------------ lib/package_manager.sh | 4 +++- lib/pip.sh | 13 ++++++++----- lib/pipenv.sh | 2 +- lib/utils.sh | 16 ++++++++++++++-- 6 files changed, 30 insertions(+), 25 deletions(-) diff --git a/bin/compile b/bin/compile index 4c2d271c3..ce78b794c 100755 --- a/bin/compile +++ b/bin/compile @@ -19,6 +19,7 @@ ENV_DIR="${3}" BUILDPACK_DIR=$(cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" && pwd) source "${BUILDPACK_DIR}/bin/utils" +source "${BUILDPACK_DIR}/lib/utils.sh" source "${BUILDPACK_DIR}/lib/cache.sh" source "${BUILDPACK_DIR}/lib/hooks.sh" source "${BUILDPACK_DIR}/lib/metadata.sh" @@ -27,7 +28,6 @@ source "${BUILDPACK_DIR}/lib/package_manager.sh" source "${BUILDPACK_DIR}/lib/pip.sh" source "${BUILDPACK_DIR}/lib/pipenv.sh" source "${BUILDPACK_DIR}/lib/python_version.sh" -source "${BUILDPACK_DIR}/lib/utils.sh" compile_start_time=$(nowms) @@ -75,6 +75,7 @@ PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh" EXPORT_PATH="${BUILDPACK_DIR}/export" GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh" WEB_CONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/WEB_CONCURRENCY.sh" +python_home='/app/.heroku/python' export PATH="/app/.heroku/python/bin:${PATH}" # Tell Python to not buffer it's stdin/stdout. @@ -156,14 +157,13 @@ meta_time "python_install_duration" "${install_python_start_time}" # Install the package manager and related tools. package_manager_install_start_time=$(nowms) -bundled_pip_module_path="$(utils::bundled_pip_module_path "${BUILD_DIR}")" case "${package_manager}" in pip) - pip::install_pip_setuptools_wheel "${bundled_pip_module_path}" "${python_major_version}" + pip::install_pip_setuptools_wheel "${python_home}" "${python_major_version}" ;; pipenv) # TODO: Stop installing pip when using Pipenv. - pip::install_pip_setuptools_wheel "${bundled_pip_module_path}" "${python_major_version}" + pip::install_pip_setuptools_wheel "${python_home}" "${python_major_version}" pipenv::install_pipenv ;; *) diff --git a/bin/utils b/bin/utils index c2b25254b..40f16a29b 100755 --- a/bin/utils +++ b/bin/utils @@ -41,15 +41,3 @@ is_module_available() { local module_name="${1}" python -c "import sys, importlib.util; sys.exit(0 if importlib.util.find_spec('${module_name}') else 1)" } - -# The requirement versions are effectively buildpack constants, however, we want -# Dependabot to be able to update them, which requires that they be in requirements -# files. The requirements files contain contents like `package==1.2.3` (and not just -# the package version) so we have to extract the version substring from it. -get_requirement_version() { - local package_name="${1}" - local requirement - requirement=$(cat "${BUILDPACK_DIR}/requirements/${package_name}.txt") - local requirement_version="${requirement#"${package_name}=="}" - echo "${requirement_version}" -} diff --git a/lib/package_manager.sh b/lib/package_manager.sh index 4c9fdd7f6..a06d212e0 100644 --- a/lib/package_manager.sh +++ b/lib/package_manager.sh @@ -36,7 +36,9 @@ function package_manager::determine_package_manager() { meta_set "setup_py_only" "false" fi - case "${#package_managers_found[@]}" in + local num_package_managers_found=${#package_managers_found[@]} + + case "${num_package_managers_found}" in 1) echo "${package_managers_found[0]}" return 0 diff --git a/lib/pip.sh b/lib/pip.sh index e58cea5af..1882f7f75 100644 --- a/lib/pip.sh +++ b/lib/pip.sh @@ -4,15 +4,18 @@ # however, it helps Shellcheck realise the options under which these functions will run. set -euo pipefail -PIP_VERSION=$(get_requirement_version 'pip') -SETUPTOOLS_VERSION=$(get_requirement_version 'setuptools') -WHEEL_VERSION=$(get_requirement_version 'wheel') +PIP_VERSION=$(utils::get_requirement_version 'pip') +SETUPTOOLS_VERSION=$(utils::get_requirement_version 'setuptools') +WHEEL_VERSION=$(utils::get_requirement_version 'wheel') function pip::install_pip_setuptools_wheel() { + local python_home="${1}" + local python_major_version="${2}" + # We use the pip wheel bundled within Python's standard library to install our chosen # pip version, since it's faster than `ensurepip` followed by an upgrade in place. - local bundled_pip_module_path="${1}" - local python_major_version="${2}" + local bundled_pip_module_path + bundled_pip_module_path="$(utils::bundled_pip_module_path "${python_home}")" meta_set "pip_version" "${PIP_VERSION}" diff --git a/lib/pipenv.sh b/lib/pipenv.sh index cbd524a61..321b0a483 100644 --- a/lib/pipenv.sh +++ b/lib/pipenv.sh @@ -4,7 +4,7 @@ # however, it helps Shellcheck realise the options under which these functions will run. set -euo pipefail -PIPENV_VERSION=$(get_requirement_version 'pipenv') +PIPENV_VERSION=$(utils::get_requirement_version 'pipenv') # TODO: Either enable or remove these. # export CLINT_FORCE_COLOR=1 diff --git a/lib/utils.sh b/lib/utils.sh index 8a0dd4a16..c855ade3e 100644 --- a/lib/utils.sh +++ b/lib/utils.sh @@ -4,17 +4,29 @@ # however, it helps Shellcheck realise the options under which these functions will run. set -euo pipefail +# The requirement versions are effectively buildpack constants, however, we want +# Dependabot to be able to update them, which requires that they be in requirements +# files. The requirements files contain contents like `package==1.2.3` (and not just +# the package version) so we have to extract the version substring from it. +function utils::get_requirement_version() { + local package_name="${1}" + local requirement + requirement=$(cat "${BUILDPACK_DIR:?}/requirements/${package_name}.txt") + local requirement_version="${requirement#"${package_name}=="}" + echo "${requirement_version}" +} + # Python bundles pip within its standard library, which we can use to install our chosen # pip version from PyPI, saving us from having to download the usual pip bootstrap script. function utils::bundled_pip_module_path() { - local build_dir="${1}" + local python_home="${1}" # We have to use a glob since the bundled wheel filename contains the pip version, which # differs between Python versions. We also have to handle the case where there are multiple # matching pip wheels, since in some versions of Python (eg 3.9.0) multiple versions of pip # were accidentally bundled upstream. Note: This implementation relies upon `nullglob` being # set, which is the case thanks to the `bin/utils` that was run earlier. - local bundled_pip_wheel_list=("${build_dir}"/.heroku/python/lib/python*/ensurepip/_bundled/pip-*.whl) + local bundled_pip_wheel_list=("${python_home}"/lib/python*/ensurepip/_bundled/pip-*.whl) local bundled_pip_wheel="${bundled_pip_wheel_list[0]}" if [[ -z "${bundled_pip_wheel}" ]]; then