From f005cd3a151585c0bafe6ec050c99ed7282875a8 Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 30 Aug 2024 11:38:54 +0000 Subject: [PATCH] Improve pip package manager tests (#256) Whilst writing the tests for the upcoming Poetry support, I made a few changes to the overall package manager testing strategy (such as using a testing buildpack to verify that at build time the tools and env vars are configured correctly) - which I've split out of the later PRs for easier review. In particular, the new testing buildpack added here is what helped me debug and locate this upstream lifecycle build time env vars bug (which would have broken apps when we switch to venvs shortly): https://github.com/buildpacks/lifecycle/issues/1393 GUS-W-16617242. --- tests/fixtures/pip_basic/requirements.txt | 2 +- .../requirements.txt | 2 +- tests/fixtures/testing_buildpack/bin/build | 24 ++ tests/fixtures/testing_buildpack/bin/detect | 3 + .../fixtures/testing_buildpack/buildpack.toml | 6 + tests/mod.rs | 8 +- tests/pip_test.rs | 300 ++++++++++-------- 7 files changed, 198 insertions(+), 147 deletions(-) create mode 100755 tests/fixtures/testing_buildpack/bin/build create mode 100755 tests/fixtures/testing_buildpack/bin/detect create mode 100644 tests/fixtures/testing_buildpack/buildpack.toml diff --git a/tests/fixtures/pip_basic/requirements.txt b/tests/fixtures/pip_basic/requirements.txt index 21e4839..eec3a22 100644 --- a/tests/fixtures/pip_basic/requirements.txt +++ b/tests/fixtures/pip_basic/requirements.txt @@ -1,2 +1,2 @@ # This package has been picked since it has no dependencies and is small/fast to install. -typing-extensions==4.7.1 +typing-extensions==4.12.2 diff --git a/tests/fixtures/pip_editable_git_compiled/requirements.txt b/tests/fixtures/pip_editable_git_compiled/requirements.txt index a9b27b0..240baa8 100644 --- a/tests/fixtures/pip_editable_git_compiled/requirements.txt +++ b/tests/fixtures/pip_editable_git_compiled/requirements.txt @@ -8,4 +8,4 @@ # The URL to the package is specified via env var, to test that user-provided env vars # are propagated to pip for use by its env var interpolation feature. --e git+${WHEEL_PACKAGE_URL}@0.40.0#egg=extension.dist&subdirectory=tests/testdata/extension.dist +-e git+${WHEEL_PACKAGE_URL}@0.44.0#egg=extension.dist&subdirectory=tests/testdata/extension.dist diff --git a/tests/fixtures/testing_buildpack/bin/build b/tests/fixtures/testing_buildpack/bin/build new file mode 100755 index 0000000..cca2d63 --- /dev/null +++ b/tests/fixtures/testing_buildpack/bin/build @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Check that: +# - The correct env vars are set for later buildpacks. +# - Python's sys.path is correct. +# - The correct version of pip was installed. +# - Both the package manager and Python can find the typing-extensions package. +# - The system site-packages directory is protected against running 'pip install' +# without having passed '--user'. +# - The typing-extensions package was installed into a separate dependencies layer. + +set -euo pipefail + +echo +echo "## Testing buildpack ##" + +printenv | sort | grep -vE '^(_|CNB_.+|HOME|HOSTNAME|OLDPWD|PWD|SHLVL)=' +echo +python -c 'import pprint, sys; pprint.pp(sys.path)' +echo +pip --version +pip list +pip install --dry-run typing-extensions +python -c 'import typing_extensions; print(typing_extensions)' diff --git a/tests/fixtures/testing_buildpack/bin/detect b/tests/fixtures/testing_buildpack/bin/detect new file mode 100755 index 0000000..0fdcf09 --- /dev/null +++ b/tests/fixtures/testing_buildpack/bin/detect @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +exit 0 diff --git a/tests/fixtures/testing_buildpack/buildpack.toml b/tests/fixtures/testing_buildpack/buildpack.toml new file mode 100644 index 0000000..18aae60 --- /dev/null +++ b/tests/fixtures/testing_buildpack/buildpack.toml @@ -0,0 +1,6 @@ +api = "0.11" + +[buildpack] +id = "testing-buildpack" +version = "0.0.0" +clear-env = true diff --git a/tests/mod.rs b/tests/mod.rs index c0f7319..af80d40 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -3,16 +3,16 @@ //! These tests are not run via automatic integration test discovery, but instead are //! imported in main.rs so that they have access to private APIs (see comment in main.rs). -use libcnb_test::BuildConfig; -use std::env; -use std::path::Path; - mod detect_test; mod django_test; mod package_manager_test; mod pip_test; mod python_version_test; +use libcnb_test::BuildConfig; +use std::env; +use std::path::Path; + const LATEST_PYTHON_3_7: &str = "3.7.17"; const LATEST_PYTHON_3_8: &str = "3.8.19"; const LATEST_PYTHON_3_9: &str = "3.9.19"; diff --git a/tests/pip_test.rs b/tests/pip_test.rs index 436043d..983d7d3 100644 --- a/tests/pip_test.rs +++ b/tests/pip_test.rs @@ -1,12 +1,17 @@ use crate::packaging_tool_versions::PIP_VERSION; -use crate::tests::{default_build_config, DEFAULT_PYTHON_VERSION}; +use crate::tests::{default_build_config, DEFAULT_PYTHON_VERSION, LATEST_PYTHON_3_11}; use indoc::{formatdoc, indoc}; use libcnb_test::{assert_contains, assert_empty, BuildpackReference, PackResult, TestRunner}; #[test] #[ignore = "integration test"] +#[allow(clippy::too_many_lines)] fn pip_basic_install_and_cache_reuse() { - let config = default_build_config("tests/fixtures/pip_basic"); + let mut config = default_build_config("tests/fixtures/pip_basic"); + config.buildpacks(vec![ + BuildpackReference::CurrentCrate, + BuildpackReference::Other("file://tests/fixtures/testing_buildpack".to_string()), + ]); TestRunner::default().build(&config, |context| { assert_empty!(context.pack_stderr); @@ -23,36 +28,61 @@ fn pip_basic_install_and_cache_reuse() { [Installing dependencies using pip] Running pip install - Collecting typing-extensions==4.7.1 (from -r requirements.txt (line 2)) - Downloading typing_extensions-4.7.1-py3-none-any.whl.metadata (3.1 kB) - Downloading typing_extensions-4.7.1-py3-none-any.whl (33 kB) + Collecting typing-extensions==4.12.2 (from -r requirements.txt (line 2)) + Downloading typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB) + Downloading typing_extensions-4.12.2-py3-none-any.whl (37 kB) Installing collected packages: typing-extensions - Successfully installed typing-extensions-4.7.1 + Successfully installed typing-extensions-4.12.2 + + ## Testing buildpack ## + CPATH=/layers/heroku_python/python/include/python3.12:/layers/heroku_python/python/include + LANG=C.UTF-8 + LD_LIBRARY_PATH=/layers/heroku_python/python/lib:/layers/heroku_python/dependencies/lib + LIBRARY_PATH=/layers/heroku_python/python/lib:/layers/heroku_python/dependencies/lib + PATH=/layers/heroku_python/python/bin:/layers/heroku_python/dependencies/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + PIP_CACHE_DIR=/layers/heroku_python/pip-cache + PIP_DISABLE_PIP_VERSION_CHECK=1 + PKG_CONFIG_PATH=/layers/heroku_python/python/lib/pkgconfig + PYTHONHOME=/layers/heroku_python/python + PYTHONUNBUFFERED=1 + PYTHONUSERBASE=/layers/heroku_python/dependencies + SOURCE_DATE_EPOCH=315532801 + + ['', + '/layers/heroku_python/python/lib/python312.zip', + '/layers/heroku_python/python/lib/python3.12', + '/layers/heroku_python/python/lib/python3.12/lib-dynload', + '/layers/heroku_python/dependencies/lib/python3.12/site-packages', + '/layers/heroku_python/python/lib/python3.12/site-packages'] + + pip {PIP_VERSION} from /layers/heroku_python/python/lib/python3.12/site-packages/pip (python 3.12) + Package Version + ----------------- ------- + pip {PIP_VERSION} + typing_extensions 4.12.2 + Defaulting to user installation because normal site-packages is not writeable + Requirement already satisfied: typing-extensions in /layers/heroku_python/dependencies/lib/python3.12/site-packages (4.12.2) + "} ); - // Check that: - // - The correct env vars are set at run-time. - // - pip is available at run-time too (and not just during the build). - // - The correct version of pip was installed. - // - pip uses (via 'PYTHONUSERBASE') the user site-packages in the dependencies - // layer, and so can find the typing-extensions package installed there. - // - The "pip update available" warning is not shown (since it should be suppressed). - // - The system site-packages directory is protected against running 'pip install' - // without having passed '--user'. + // Check that at run-time: + // - The correct env vars are set. + // - pip is available (rather than just during the build). + // - Both pip and Python can find the typing-extensions package. let command_output = context.run_shell_command( indoc! {" set -euo pipefail printenv | sort | grep -vE '^(_|HOME|HOSTNAME|OLDPWD|PWD|SHLVL)=' echo pip list - pip install --dry-run typing-extensions + python -c 'import typing_extensions' "} ); assert_empty!(command_output.stderr); - assert_contains!( + assert_eq!( command_output.stdout, - &formatdoc! {" + formatdoc! {" LANG=C.UTF-8 LD_LIBRARY_PATH=/layers/heroku_python/python/lib:/layers/heroku_python/dependencies/lib PATH=/layers/heroku_python/dependencies/bin:/layers/heroku_python/python/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin @@ -64,10 +94,8 @@ fn pip_basic_install_and_cache_reuse() { Package Version ----------------- ------- pip {PIP_VERSION} - typing_extensions 4.7.1 - Defaulting to user installation because normal site-packages is not writeable - Requirement already satisfied: typing-extensions in /layers/heroku_python/dependencies/lib/" - } + typing_extensions 4.12.2 + "} ); context.rebuild(&config, |rebuild_context| { @@ -85,101 +113,93 @@ fn pip_basic_install_and_cache_reuse() { [Installing dependencies using pip] Using cached pip download/wheel cache Running pip install - Collecting typing-extensions==4.7.1 (from -r requirements.txt (line 2)) - Using cached typing_extensions-4.7.1-py3-none-any.whl.metadata (3.1 kB) - Using cached typing_extensions-4.7.1-py3-none-any.whl (33 kB) + Collecting typing-extensions==4.12.2 (from -r requirements.txt (line 2)) + Using cached typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB) + Using cached typing_extensions-4.12.2-py3-none-any.whl (37 kB) Installing collected packages: typing-extensions - Successfully installed typing-extensions-4.7.1 + Successfully installed typing-extensions-4.12.2 "} ); }); }); } -// This tests that: -// - The cached layers are correctly invalidated when Python/other versions change. -// - The layer metadata written by older versions of the buildpack are still compatible. #[test] #[ignore = "integration test"] -fn pip_cache_invalidation_with_compatible_metadata() { - let config = default_build_config("tests/fixtures/pip_basic"); +fn pip_cache_invalidation_python_version_changed() { + let config = default_build_config("tests/fixtures/python_3.11"); + let rebuild_config = default_build_config("tests/fixtures/pip_basic"); - TestRunner::default().build( - config.clone().buildpacks([BuildpackReference::Other( - "docker://docker.io/heroku/buildpack-python:0.14.0".to_string(), - )]), - |context| { - context.rebuild(config, |rebuild_context| { - assert_empty!(rebuild_context.pack_stderr); - assert_contains!( - rebuild_context.pack_stdout, - &formatdoc! {" - [Determining Python version] - No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. - To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes - - [Installing Python and pip] - Discarding cache since: - - The Python version has changed from 3.12.4 to {DEFAULT_PYTHON_VERSION} - - The pip version has changed from 24.1.2 to {PIP_VERSION} - Installing Python {DEFAULT_PYTHON_VERSION} - Installing pip {PIP_VERSION} - - [Installing dependencies using pip] - Discarding cached pip download/wheel cache - Running pip install - Collecting typing-extensions==4.7.1 (from -r requirements.txt (line 2)) - Downloading typing_extensions-4.7.1-py3-none-any.whl.metadata (3.1 kB) - Downloading typing_extensions-4.7.1-py3-none-any.whl (33 kB) - Installing collected packages: typing-extensions - Successfully installed typing-extensions-4.7.1 - "} - ); - }); - }, - ); + TestRunner::default().build(config, |context| { + context.rebuild(rebuild_config, |rebuild_context| { + assert_empty!(rebuild_context.pack_stderr); + assert_contains!( + rebuild_context.pack_stdout, + &formatdoc! {" + [Determining Python version] + No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. + To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes + + [Installing Python and pip] + Discarding cache since: + - The Python version has changed from {LATEST_PYTHON_3_11} to {DEFAULT_PYTHON_VERSION} + Installing Python {DEFAULT_PYTHON_VERSION} + Installing pip {PIP_VERSION} + + [Installing dependencies using pip] + Discarding cached pip download/wheel cache + Running pip install + Collecting typing-extensions==4.12.2 (from -r requirements.txt (line 2)) + Downloading typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB) + Downloading typing_extensions-4.12.2-py3-none-any.whl (37 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.12.2 + "} + ); + }); + }); } -// This tests that: -// - The cached layers are correctly invalidated when the layer metadata was incompatible. -// - That a suitable message was output explaining why. +// This tests that cached layers from a previous buildpack version are compatible, or if we've +// decided to break compatibility recently, that the layers are at least invalidated gracefully. #[test] #[ignore = "integration test"] -fn pip_cache_invalidation_with_incompatible_metadata() { - let config = default_build_config("tests/fixtures/pip_basic"); +fn pip_cache_previous_buildpack_version() { + let mut config = default_build_config("tests/fixtures/pip_basic"); + config.buildpacks([BuildpackReference::Other( + "docker://docker.io/heroku/buildpack-python:0.14.0".to_string(), + )]); + let rebuild_config = default_build_config("tests/fixtures/pip_basic"); - TestRunner::default().build( - config.clone().buildpacks([BuildpackReference::Other( - "docker://docker.io/heroku/buildpack-python:0.13.0".to_string(), - )]), - |context| { - context.rebuild(config, |rebuild_context| { - assert_empty!(rebuild_context.pack_stderr); - assert_contains!( - rebuild_context.pack_stdout, - &formatdoc! {" - [Determining Python version] - No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. - To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes - - [Installing Python and pip] - Discarding cache since the buildpack cache format has changed - Installing Python {DEFAULT_PYTHON_VERSION} - Installing pip {PIP_VERSION} - - [Installing dependencies using pip] - Discarding cached pip download/wheel cache - Running pip install - Collecting typing-extensions==4.7.1 (from -r requirements.txt (line 2)) - Downloading typing_extensions-4.7.1-py3-none-any.whl.metadata (3.1 kB) - Downloading typing_extensions-4.7.1-py3-none-any.whl (33 kB) - Installing collected packages: typing-extensions - Successfully installed typing-extensions-4.7.1 - "} - ); - }); - }, - ); + TestRunner::default().build(config, |context| { + context.rebuild(rebuild_config, |rebuild_context| { + assert_empty!(rebuild_context.pack_stderr); + assert_contains!( + rebuild_context.pack_stdout, + &formatdoc! {" + [Determining Python version] + No Python version specified, using the current default of Python {DEFAULT_PYTHON_VERSION}. + To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes + + [Installing Python and pip] + Discarding cache since: + - The Python version has changed from 3.12.4 to {DEFAULT_PYTHON_VERSION} + - The pip version has changed from 24.1.2 to {PIP_VERSION} + Installing Python {DEFAULT_PYTHON_VERSION} + Installing pip {PIP_VERSION} + + [Installing dependencies using pip] + Discarding cached pip download/wheel cache + Running pip install + Collecting typing-extensions==4.12.2 (from -r requirements.txt (line 2)) + Downloading typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB) + Downloading typing_extensions-4.12.2-py3-none-any.whl (37 kB) + Installing collected packages: typing-extensions + Successfully installed typing-extensions-4.12.2 + "} + ); + }); + }); } // This tests that: @@ -191,48 +211,46 @@ fn pip_cache_invalidation_with_incompatible_metadata() { #[test] #[ignore = "integration test"] fn pip_editable_git_compiled() { - TestRunner::default().build( - default_build_config( "tests/fixtures/pip_editable_git_compiled") - .env("WHEEL_PACKAGE_URL", "https://github.com/pypa/wheel"), - |context| { - assert_contains!( - context.pack_stdout, - "Cloning https://github.com/pypa/wheel (to revision 0.40.0) to /layers/heroku_python/dependencies/src/extension-dist" - ); - }, - ); + let mut config = default_build_config("tests/fixtures/pip_editable_git_compiled"); + config.env("WHEEL_PACKAGE_URL", "https://github.com/pypa/wheel.git"); + + TestRunner::default().build(config, |context| { + assert_contains!( + context.pack_stdout, + "Cloning https://github.com/pypa/wheel.git (to revision 0.44.0) to /layers/heroku_python/dependencies/src/extension-dist" + ); + }); } #[test] #[ignore = "integration test"] fn pip_install_error() { - TestRunner::default().build( - default_build_config( "tests/fixtures/pip_invalid_requirement") - .expected_pack_result(PackResult::Failure), - |context| { - // Ideally we could test a combined stdout/stderr, however libcnb-test doesn't support this: - // https://github.com/heroku/libcnb.rs/issues/536 - assert_contains!( - context.pack_stdout, - indoc! {" - [Installing dependencies using pip] - Running pip install - "} - ); - assert_contains!( - context.pack_stderr, - indoc! {" - ERROR: Invalid requirement: 'an-invalid-requirement!': Expected end or semicolon (after name and no valid version specifier) - an-invalid-requirement! - ^ (from line 1 of requirements.txt) - - [Error: Unable to install dependencies using pip] - The 'pip install' command to install the application's dependencies from - 'requirements.txt' failed (exit status: 1). - - See the log output above for more information. - "} - ); - }, - ); + let mut config = default_build_config("tests/fixtures/pip_invalid_requirement"); + config.expected_pack_result(PackResult::Failure); + + TestRunner::default().build(config, |context| { + // Ideally we could test a combined stdout/stderr, however libcnb-test doesn't support this: + // https://github.com/heroku/libcnb.rs/issues/536 + assert_contains!( + context.pack_stdout, + indoc! {" + [Installing dependencies using pip] + Running pip install + "} + ); + assert_contains!( + context.pack_stderr, + indoc! {" + ERROR: Invalid requirement: 'an-invalid-requirement!': Expected end or semicolon (after name and no valid version specifier) + an-invalid-requirement! + ^ (from line 1 of requirements.txt) + + [Error: Unable to install dependencies using pip] + The 'pip install' command to install the application's dependencies from + 'requirements.txt' failed (exit status: 1). + + See the log output above for more information. + "} + ); + }); }