From 80abbe3e5dc9b267f1800e8942e05fb21cab5eef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 12 Jan 2025 15:24:20 +0100 Subject: [PATCH 01/11] Add trusted publisher release workfiow --- .github/workflows/release.yml | 46 +++++++++++++++++++++++ docs/html/development/release-process.rst | 7 +--- 2 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..aa87fa91584 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,46 @@ +name: Publish Python 🐍 distribution 📦 to PyPI + +on: + push: + tags: + - "*" + +jobs: + build: + name: Build distribution 📦 + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Build a binary wheel and a source tarball + run: pipx run build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/pip/${{ github.ref_name }} + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/docs/html/development/release-process.rst b/docs/html/development/release-process.rst index 5bf0d278b71..77ee9b5d46b 100644 --- a/docs/html/development/release-process.rst +++ b/docs/html/development/release-process.rst @@ -146,11 +146,8 @@ Creating a new release This will update the relevant files and tag the correct commit. #. Submit the ``release/YY.N`` branch as a pull request and ensure CI passes. Merge the changes back into ``main`` and pull them back locally. -#. Build the release artifacts using ``nox -s build-release -- YY.N``. - This will checkout the tag, generate the distribution files to be - uploaded and checkout the main branch again. -#. Upload the release to PyPI using ``nox -s upload-release -- YY.N``. -#. Push the tag created by ``prepare-release``. +#. Push the tag created by ``prepare-release``. This will trigger the release + workflow on GitHub and publish to PyPI. #. Regenerate the ``get-pip.py`` script in the `get-pip repository`_ (as documented there) and commit the results. #. Submit a Pull Request to `CPython`_ adding the new version of pip From 2c5ff943b2f815f5f4a2e59c0e47b2788b224efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 12 Jan 2025 15:40:33 +0100 Subject: [PATCH 02/11] Use pinned build dpendencies in the release workflow --- .github/workflows/release.yml | 9 ++++----- MANIFEST.in | 3 +++ build-requirements.in | 2 ++ build-requirements.txt | 24 ++++++++++++++++++++++++ 4 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 build-requirements.in create mode 100644 build-requirements.txt diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aa87fa91584..f82467f57ec 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,12 +12,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.x" - name: Build a binary wheel and a source tarball - run: pipx run build + run: | + python3 -m venv build-env + build-env/bin/python -m pip install --no-deps --require-hashes -r build-requirements.txt + build-env/bin/python -m build --no-isolation - name: Store the distribution packages uses: actions/upload-artifact@v4 with: diff --git a/MANIFEST.in b/MANIFEST.in index 6f4197565d3..98b43257057 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,6 +5,9 @@ include README.rst include SECURITY.md include pyproject.toml +include build-requirements.in +include build-requirements.txt + include src/pip/_vendor/README.rst include src/pip/_vendor/vendor.txt recursive-include src/pip/_vendor *LICENSE* diff --git a/build-requirements.in b/build-requirements.in new file mode 100644 index 00000000000..4bc215a28d0 --- /dev/null +++ b/build-requirements.in @@ -0,0 +1,2 @@ +build +setuptools diff --git a/build-requirements.txt b/build-requirements.txt new file mode 100644 index 00000000000..ad876c4ada0 --- /dev/null +++ b/build-requirements.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes build-requirements.in +# +build==1.2.2.post1 \ + --hash=sha256:1d61c0887fa860c01971625baae8bdd338e517b836a2f70dd1f7aa3a6b2fc5b5 \ + --hash=sha256:b36993e92ca9375a219c99e606a122ff365a760a2d4bba0caa09bd5278b608b7 + # via -r build-requirements.in +packaging==24.2 \ + --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \ + --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f + # via build +pyproject-hooks==1.2.0 \ + --hash=sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8 \ + --hash=sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913 + # via build + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.8.0 \ + --hash=sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6 \ + --hash=sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3 + # via -r build-requirements.in From 21fbe62a321db2ee353e935d808b05c4fd5cbf3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 12 Jan 2025 15:49:57 +0100 Subject: [PATCH 03/11] Pin GitHub actions used for release --- .github/workflows/release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f82467f57ec..d5b8c1c09fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,14 +11,14 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Build a binary wheel and a source tarball run: | python3 -m venv build-env - build-env/bin/python -m pip install --no-deps --require-hashes -r build-requirements.txt + build-env/bin/python -m pip install --no-deps --only-binary :all: --require-hashes -r build-requirements.txt build-env/bin/python -m build --no-isolation - name: Store the distribution packages - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 with: name: python-package-distributions path: dist/ @@ -37,9 +37,9 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v4 + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 with: name: python-package-distributions path: dist/ - name: Publish distribution 📦 to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # release/v1 From 1d270e75ef7f09ee61111494714156da7de53e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 14 Jan 2025 14:48:31 +0100 Subject: [PATCH 04/11] Factor out build script also, set SOURCE_DATE_EPOCH, and use python -I --- .github/workflows/release.yml | 5 +-- MANIFEST.in | 1 + build-project.py | 62 +++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100755 build-project.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d5b8c1c09fe..84c3f5366db 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,10 +13,7 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 - name: Build a binary wheel and a source tarball - run: | - python3 -m venv build-env - build-env/bin/python -m pip install --no-deps --only-binary :all: --require-hashes -r build-requirements.txt - build-env/bin/python -m build --no-isolation + run: ./build-project.py - name: Store the distribution packages uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 with: diff --git a/MANIFEST.in b/MANIFEST.in index 98b43257057..cb8e14df96b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,7 @@ include pyproject.toml include build-requirements.in include build-requirements.txt +include build-project.py include src/pip/_vendor/README.rst include src/pip/_vendor/vendor.txt diff --git a/build-project.py b/build-project.py new file mode 100755 index 00000000000..a89eec16296 --- /dev/null +++ b/build-project.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Build pip using pinned build requirements.""" + +import subprocess +import sys +import tempfile +from pathlib import Path + + +def get_git_head_timestamp() -> str: + return subprocess.run( + [ + "git", + "log", + "-1", + "--pretty=format:%ct", + ], + text=True, + stdout=subprocess.PIPE, + ).stdout.strip() + + +def main() -> None: + with tempfile.TemporaryDirectory() as build_env: + subprocess.run( + [ + sys.executable, + "-m", + "venv", + build_env, + ], + check=True, + ) + build_python = Path(build_env) / "bin" / "python" + subprocess.run( + [ + build_python, + "-Im", + "pip", + "install", + "--no-deps", + "--only-binary=:all:", + "--require-hashes", + "-r", + Path(__file__).parent / "build-requirements.txt", + ], + check=True, + ) + subprocess.run( + [ + build_python, + "-Im", + "build", + "--no-isolation", + ], + check=True, + env={"SOURCE_DATE_EPOCH": get_git_head_timestamp()}, + ) + + +if __name__ == "__main__": + main() From e968c6174bd5ecfaa21bc96173599486fb02dc03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 14 Jan 2025 15:13:00 +0100 Subject: [PATCH 05/11] Update nox session to use new build script --- noxfile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index 6e6e144bccb..70e24a01142 100644 --- a/noxfile.py +++ b/noxfile.py @@ -339,7 +339,7 @@ def build_release(session: nox.Session) -> None: ) session.log("# Install dependencies") - session.install("build", "twine") + session.install("twine") with release.isolated_temporary_checkout(session, version) as build_dir: session.log( @@ -375,11 +375,11 @@ def build_dists(session: nox.Session) -> List[str]: ) session.log("# Build distributions") - session.run("python", "-m", "build", silent=True) + session.run("python", "build-project.py", silent=True) produced_dists = glob.glob("dist/*") session.log(f"# Verify distributions: {', '.join(produced_dists)}") - session.run("twine", "check", *produced_dists, silent=True) + session.run("twine", "check", "--strict", *produced_dists, silent=True) return produced_dists From f22ff81ba7845132949df17819b5c81a00f5ba8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 14 Jan 2025 15:19:08 +0100 Subject: [PATCH 06/11] Add news --- news/13048.process.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/13048.process.rst diff --git a/news/13048.process.rst b/news/13048.process.rst new file mode 100644 index 00000000000..a6c03018998 --- /dev/null +++ b/news/13048.process.rst @@ -0,0 +1 @@ +Use PyPI trusted publisher workflow for release. From 4c389522e67a5656fb250381de44aef7c6f473e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 14 Jan 2025 15:29:36 +0100 Subject: [PATCH 07/11] Add dependabot config for build requirements --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2390d8c809e..a3a55e8b329 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,3 +8,7 @@ updates: github-actions: patterns: - "*" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" From 48fb3d541b7120a2b88a2618dc227402cf29cebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Tue, 14 Jan 2025 16:40:59 +0100 Subject: [PATCH 08/11] Update news MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- news/13048.process.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/news/13048.process.rst b/news/13048.process.rst index a6c03018998..b4d18461b7a 100644 --- a/news/13048.process.rst +++ b/news/13048.process.rst @@ -1 +1,3 @@ -Use PyPI trusted publisher workflow for release. +Started releasing to PyPI from a GitHub Actions CI/CD workflow that implements trusted publishing and bundles :pep:`740` digital attestations. + +In addition to being signed, the released distribution packages are now reproducible through the commit timestamp. From b6e5f3fc31f0b2b1af7c964fa6c874fac021dcc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Wed, 15 Jan 2025 11:24:45 +0100 Subject: [PATCH 09/11] Add persist-credential: false to release workflow Following recommendation from zizmor --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 84c3f5366db..9506eeb3304 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,6 +12,8 @@ jobs: steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + persist-credentials: false - name: Build a binary wheel and a source tarball run: ./build-project.py - name: Store the distribution packages From 08fe349a35ae759f0fbf988a99e5794ac55bafd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Wed, 15 Jan 2025 13:09:51 +0100 Subject: [PATCH 10/11] Run dependebot weekly to get feedback sooner We can can back to monthly later when we have a better understanding of how it works. --- .github/dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a3a55e8b329..456596841a9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,7 +3,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "monthly" + interval: "weekly" groups: github-actions: patterns: @@ -11,4 +11,4 @@ updates: - package-ecosystem: "pip" directory: "/" schedule: - interval: "monthly" + interval: "weekly" From 1801d83b7a81fea1ac0bd36c6184398315c74117 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= Date: Sun, 19 Jan 2025 12:43:00 +0100 Subject: [PATCH 11/11] Make build-project.py portable --- build-project.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/build-project.py b/build-project.py index a89eec16296..78e183da08a 100755 --- a/build-project.py +++ b/build-project.py @@ -2,9 +2,22 @@ """Build pip using pinned build requirements.""" import subprocess -import sys import tempfile +import venv +from os import PathLike from pathlib import Path +from types import SimpleNamespace + + +class EnvBuilder(venv.EnvBuilder): + """A subclass of venv.EnvBuilder that exposes the python executable command.""" + + def ensure_directories( + self, env_dir: str | bytes | PathLike[str] | PathLike[bytes] + ) -> SimpleNamespace: + context = super().ensure_directories(env_dir) + self.env_exec_cmd = context.env_exec_cmd + return context def get_git_head_timestamp() -> str: @@ -22,19 +35,11 @@ def get_git_head_timestamp() -> str: def main() -> None: with tempfile.TemporaryDirectory() as build_env: + env_builder = EnvBuilder(with_pip=True) + env_builder.create(build_env) subprocess.run( [ - sys.executable, - "-m", - "venv", - build_env, - ], - check=True, - ) - build_python = Path(build_env) / "bin" / "python" - subprocess.run( - [ - build_python, + env_builder.env_exec_cmd, "-Im", "pip", "install", @@ -48,7 +53,7 @@ def main() -> None: ) subprocess.run( [ - build_python, + env_builder.env_exec_cmd, "-Im", "build", "--no-isolation",