feat: support parsing whl METADATA on any python version (#1693)

With this PR we can deterministically parse the METADATA and generate a
`BUILD.bazel` file using the config settings introduced in #1555. Let's
imagine we had a `requirements.txt` file that used only wheels, we could
use the host interpreter to parse the wheel metadata for all the target
platforms and use the version aware toolchain at runtime. This
potentially
unlocks more clever layouts of the `bzlmod` hub repos explored in #1625
where we could have a single `whl_library` instance for all versions
within
a single hub repo.

Work towards #1643.

CHANGELOG.md

@@ -35,6 +35,15 @@ A brief description of the categories of changes:
 
 * (py_wheel) Added `requires_file` and `extra_requires_files` attributes.
 
+* (whl_library) *experimental_target_platforms* now supports specifying the
+  Python version explicitly and the output `BUILD.bazel` file will be correct
+  irrespective of the python interpreter that is generating the file and
+  extracting the `whl` distribution. Multiple python target version can be
+  specified and the code generation will generate version specific dependency
+  closures but that is not yet ready to be used and may break the build if
+  the default python version is not selected using
+  `common --@rules_python//python/config_settings:python_version=X.Y.Z`.
+
 ## 0.29.0 - 2024-01-22
 
 [0.29.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.29.0
@@ -59,6 +68,7 @@ A brief description of the categories of changes:
   platform-specific content in `MODULE.bazel.lock` files; Follow
   [#1643](https://github.com/bazelbuild/rules_python/issues/1643) for removing
   platform-specific content in `MODULE.bazel.lock` files.
+
 * (wheel) The stamp variables inside the distribution name are no longer
   lower-cased when normalizing under PEP440 conventions.
 

examples/bzlmod/.bazelrc

@@ -1,4 +1,4 @@
-common --experimental_enable_bzlmod
+common --enable_bzlmod
 
 coverage --java_runtime_version=remotejdk_11
 

examples/bzlmod/MODULE.bazel

@@ -107,9 +107,10 @@ pip.parse(
     # You can use one of the values below to specify the target platform
     # to generate the dependency graph for.
     experimental_target_platforms = [
-        "all",
-        "linux_*",
-        "host",
+        # Specifying the target platforms explicitly
+        "cp39_linux_x86_64",
+        "cp39_linux_*",
+        "cp39_*",
     ],
     hub_name = "pip",
     python_version = "3.9",
@@ -137,8 +138,13 @@ pip.parse(
     # You can use one of the values below to specify the target platform
     # to generate the dependency graph for.
     experimental_target_platforms = [
-        "all",
+        # Using host python version
         "linux_*",
+        "osx_*",
+        "windows_*",
+        # Or specifying an exact platform
+        "linux_x86_64",
+        # Or the following to get the `host` platform only
         "host",
     ],
     hub_name = "pip",

python/pip_install/pip_repository.bzl

@@ -493,13 +493,19 @@ WARNING: It may not work as expected in cases where the python interpreter
 implementation that is being used at runtime is different between different platforms.
 This has been tested for CPython only.
 
-Special values: `all` (for generating deps for all platforms), `host` (for
-generating deps for the host platform only). `linux_*` and other `<os>_*` values.
-In the future we plan to set `all` as the default to this attribute.
-
 For specific target platforms use values of the form `<os>_<arch>` where `<os>`
 is one of `linux`, `osx`, `windows` and arch is one of `x86_64`, `x86_32`,
 `aarch64`, `s390x` and `ppc64le`.
+
+You can also target a specific Python version by using `cp3<minor_version>_<os>_<arch>`.
+If multiple python versions are specified as target platforms, then select statements
+of the `lib` and `whl` targets will include usage of version aware toolchain config 
+settings like `@rules_python//python/config_settings:is_python_3.y`.
+
+Special values: `host` (for generating deps for the host platform only) and
+`<prefix>_*` values. For example, `cp39_*`, `linux_*`, `cp39_linux_*`.
+
+NOTE: this is not for cross-compiling Python wheels but rather for parsing the `whl` METADATA correctly.
 """,
     ),
     "extra_pip_args": attr.string_list(
@@ -749,7 +755,7 @@ def _whl_library_impl(rctx):
             # NOTE @aignas 2023-12-04: if the wheel is a platform specific
             # wheel, we only include deps for that target platform
             target_platforms = [
-                "{}_{}".format(p.os, p.cpu)
+                "{}_{}_{}".format(parsed_whl.abi_tag, p.os, p.cpu)
                 for p in whl_target_platforms(parsed_whl.platform_tag)
             ]
 

python/pip_install/private/generate_whl_library_build_bazel.bzl

@@ -48,8 +48,7 @@ py_binary(
 """
 
 _BUILD_TEMPLATE = """\
-load("@rules_python//python:defs.bzl", "py_library", "py_binary")
-load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
+{loads}
 
 package(default_visibility = ["//visibility:public"])
 
@@ -102,22 +101,37 @@ alias(
 )
 """
 
+def _plat_label(plat):
+    if plat.startswith("@//"):
+        return "@@" + str(Label("//:BUILD.bazel")).partition("//")[0].strip("@") + plat.strip("@")
+    elif plat.startswith("@"):
+        return str(Label(plat))
+    else:
+        return ":is_" + plat
+
 def _render_list_and_select(deps, deps_by_platform, tmpl):
-    deps = render.list([tmpl.format(d) for d in deps])
+    deps = render.list([tmpl.format(d) for d in sorted(deps)])
 
     if not deps_by_platform:
         return deps
 
     deps_by_platform = {
-        p if p.startswith("@") else ":is_" + p: [
+        _plat_label(p): [
             tmpl.format(d)
-            for d in deps
+            for d in sorted(deps)
         ]
-        for p, deps in deps_by_platform.items()
+        for p, deps in sorted(deps_by_platform.items())
     }
 
     # Add the default, which means that we will be just using the dependencies in
     # `deps` for platforms that are not handled in a special way by the packages
+    #
+    # FIXME @aignas 2024-01-24: This currently works as expected only if the default
+    # value of the @rules_python//python/config_settings:python_version is set in
+    # the `.bazelrc`. If it is unset, then the we don't get the expected behaviour
+    # in cases where we are using a simple `py_binary` using the default toolchain
+    # without forcing any transitions. If the `python_version` config setting is set
+    # via .bazelrc, then everything works correctly.
     deps_by_platform["//conditions:default"] = []
     deps_by_platform = render.select(deps_by_platform, value_repr = render.list)
 
@@ -126,6 +140,87 @@ def _render_list_and_select(deps, deps_by_platform, tmpl):
     else:
         return "{} + {}".format(deps, deps_by_platform)
 
+def _render_config_settings(dependencies_by_platform):
+    py_version_by_os_arch = {}
+    for p in dependencies_by_platform:
+        # p can be one of the following formats:
+        # * @platforms//os:{value}
+        # * @platforms//cpu:{value}
+        # * @//python/config_settings:is_python_3.{minor_version}
+        # * {os}_{cpu}
+        # * cp3{minor_version}_{os}_{cpu}
+        if p.startswith("@"):
+            continue
+
+        abi, _, tail = p.partition("_")
+        if not abi.startswith("cp"):
+            tail = p
+            abi = ""
+        os, _, arch = tail.partition("_")
+        os = "" if os == "anyos" else os
+        arch = "" if arch == "anyarch" else arch
+
+        py_version_by_os_arch.setdefault((os, arch), []).append(abi)
+
+    if not py_version_by_os_arch:
+        return None, None
+
+    loads = []
+    additional_content = []
+    for (os, arch), abis in py_version_by_os_arch.items():
+        constraint_values = []
+        if os:
+            constraint_values.append("@platforms//os:{}".format(os))
+        if arch:
+            constraint_values.append("@platforms//cpu:{}".format(arch))
+
+        os_arch = (os or "anyos") + "_" + (arch or "anyarch")
+        additional_content.append(
+            """\
+config_setting(
+    name = "is_{name}",
+    constraint_values = {values},
+    visibility = ["//visibility:private"],
+)""".format(
+                name = os_arch,
+                values = render.indent(render.list(sorted([str(Label(c)) for c in constraint_values]))).strip(),
+            ),
+        )
+
+        if abis == [""]:
+            if not os or not arch:
+                fail("BUG: both os and arch should be set in this case")
+            continue
+
+        for abi in abis:
+            if not loads:
+                loads.append("""load("@bazel_skylib//lib:selects.bzl", "selects")""")
+            minor_version = int(abi[len("cp3"):])
+            setting = "@@{rules_python}//python/config_settings:is_python_3.{version}".format(
+                rules_python = str(Label("//:BUILD.bazel")).partition("//")[0].strip("@"),
+                version = minor_version,
+            )
+            settings = [
+                ":is_" + os_arch,
+                setting,
+            ]
+
+            plat = "{}_{}".format(abi, os_arch)
+
+            additional_content.append(
+                """\
+selects.config_setting_group(
+    name = "{name}",
+    match_all = {values},
+    visibility = ["//visibility:private"],
+)""".format(
+                    name = _plat_label(plat).lstrip(":"),
+                    values = render.indent(render.list(sorted(settings))).strip(),
+                ),
+            )
+
+    return loads, "\n\n".join(additional_content)
+
 def generate_whl_library_build_bazel(
         *,
         repo_prefix,
@@ -228,24 +323,17 @@ def generate_whl_library_build_bazel(
         if deps
     }
 
-    for p in dependencies_by_platform:
-        if p.startswith("@"):
-            continue
-
-        os, _, cpu = p.partition("_")
+    loads = [
+        """load("@rules_python//python:defs.bzl", "py_library", "py_binary")""",
+        """load("@bazel_skylib//rules:copy_file.bzl", "copy_file")""",
+    ]
 
-        additional_content.append(
-            """\
-config_setting(
-    name = "is_{os}_{cpu}",
-    constraint_values = [
-        "@platforms//cpu:{cpu}",
-        "@platforms//os:{os}",
-    ],
-    visibility = ["//visibility:private"],
-)
-""".format(os = os, cpu = cpu),
-        )
+    loads_, config_settings_content = _render_config_settings(dependencies_by_platform)
+    if config_settings_content:
+        for line in loads_:
+            if line not in loads:
+                loads.append(line)
+        additional_content.append(config_settings_content)
 
     lib_dependencies = _render_list_and_select(
         deps = dependencies,
@@ -277,6 +365,7 @@ config_setting(
     contents = "\n".join(
         [
             _BUILD_TEMPLATE.format(
+                loads = "\n".join(loads),
                 py_library_public_label = PY_LIBRARY_PUBLIC_LABEL,
                 py_library_impl_label = PY_LIBRARY_IMPL_LABEL,
                 py_library_actual_label = library_impl_label,

python/pip_install/tools/wheel_installer/arguments_test.py

@@ -58,7 +58,8 @@ def test_platform_aggregation(self) -> None:
             args=[
                 "--platform=host",
                 "--platform=linux_*",
-                "--platform=all",
+                "--platform=osx_*",
+                "--platform=windows_*",
                 "--requirement=foo",
             ]
         )