Skip to content

Commit

Permalink
fix: make sys.executable work with script bootstrap (#2409)
Browse files Browse the repository at this point in the history 
When `--bootstrap_impl=script` is used, `PYTHONPATH` is no longer used
to set the import
paths, which means subprocesses no longer inherit the Bazel paths. This
is generally a
good thing, but breaks when `sys.executable` is used to directly invoke
the interpreter.
Such an invocation assumes the interpreter will have the same packages
available and works
with the system_python bootstrap.

To fix, have the script bootstrap use a basic virtual env. This allows
it to intercept
interpreter startup even when the Bazel executable isn't invoked. Under
the hood, there's
two pieces to make this work. The first is a binary uses
`declare_symlink()` to write a
relative-path based symlink that points to the underlying Python
interpreter. The second
piece is a site init hook (triggered by a `.pth` file using an `import`
line) performs
sys.path setup as part of site (`import site`) initialization.

Fixes #2169

---------

Co-authored-by: Ignas Anikevicius <[email protected]>
  • Loading branch information
@rickeylev @aignas
rickeylev and aignas authored Nov 21, 2024
1 parent 4a55ef4 commit 8ff4386
Show file tree
Hide file tree
Showing 15 changed files with 617 additions and 159 deletions.
4 changes: 2 additions & 2 deletions .bazelrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
# (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
# To update these lines, execute
# `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered

test --test_output=errors

Expand Down
8 changes: 8 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ Unreleased changes template.
Other changes:
* (python_repository) Start honoring the `strip_prefix` field for `zstd` archives.
* (pypi) {bzl:obj}`pip_parse.extra_hub_aliases` now works in WORKSPACE files.
* (binaries/tests) For {obj}`--bootstrap_impl=script`, a binary-specific (but
otherwise empty) virtual env is used to customize `sys.path` initialization.

{#v0-0-0-fixed}
### Fixed
Expand All @@ -83,6 +85,9 @@ Other changes:
Fixes ([2337](https://github.com/bazelbuild/rules_python/issues/2337)).
* (uv): Correct the sha256sum for the `uv` binary for aarch64-apple-darwin.
Fixes ([2411](https://github.com/bazelbuild/rules_python/issues/2411)).
* (binaries/tests) ({obj}`--bootstrap_impl=scipt`) Using `sys.executable` will
use the same `sys.path` setup as the calling binary.
([2169](https://github.com/bazelbuild/rules_python/issues/2169)).

{#v0-0-0-added}
### Added
Expand All @@ -97,6 +102,9 @@ Other changes:
for the latest toolchain versions for each minor Python version. You can control
the toolchain selection by using the
{bzl:obj}`//python/config_settings:py_linux_libc` build flag.
* (providers) Added {obj}`py_runtime_info.site_init_template` and
{obj}`PyRuntimeInfo.site_init_template` for specifying the template to use to
initialize the interpreter via venv startup hooks.

{#v0-0-0-removed}
### Removed
Expand Down
8 changes: 8 additions & 0 deletions python/private/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -702,6 +702,14 @@ filegroup(
visibility = ["//visibility:public"],
)

filegroup(
name = "site_init_template",
srcs = ["site_init_template.py"],
# Not actually public. Only public because it's an implicit dependency of
# py_runtime.
visibility = ["//visibility:public"],
)

# NOTE: Windows builds don't use this bootstrap. Instead, a native Windows
# program locates some Python exe and runs `python.exe foo.zip` which
# runs the __main__.py in the zip file.
Expand Down
139 changes: 133 additions & 6 deletions python/private/py_executable_bazel.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,9 @@ the `srcs` of Python targets as required.
"_py_toolchain_type": attr.label(
default = TARGET_TOOLCHAIN_TYPE,
),
"_python_version_flag": attr.label(
default = "//python/config_settings:python_version",
),
"_windows_launcher_maker": attr.label(
default = "@bazel_tools//tools/launcher:launcher_maker",
cfg = "exec",
Expand Down Expand Up @@ -177,13 +180,22 @@ def _create_executable(
else:
base_executable_name = executable.basename

venv = None

# The check for stage2_bootstrap_template is to support legacy
# BuiltinPyRuntimeInfo providers, which is likely to come from
# @bazel_tools//tools/python:autodetecting_toolchain, the toolchain used
# for workspace builds when no rules_python toolchain is configured.
if (BootstrapImplFlag.get_value(ctx) == BootstrapImplFlag.SCRIPT and
runtime_details.effective_runtime and
hasattr(runtime_details.effective_runtime, "stage2_bootstrap_template")):
venv = _create_venv(
ctx,
output_prefix = base_executable_name,
imports = imports,
runtime_details = runtime_details,
)

stage2_bootstrap = _create_stage2_bootstrap(
ctx,
output_prefix = base_executable_name,
Expand All @@ -192,11 +204,12 @@ def _create_executable(
imports = imports,
runtime_details = runtime_details,
)
extra_runfiles = ctx.runfiles([stage2_bootstrap])
extra_runfiles = ctx.runfiles([stage2_bootstrap] + venv.files_without_interpreter)
zip_main = _create_zip_main(
ctx,
stage2_bootstrap = stage2_bootstrap,
runtime_details = runtime_details,
venv = venv,
)
else:
stage2_bootstrap = None
Expand Down Expand Up @@ -272,6 +285,7 @@ def _create_executable(
zip_file = zip_file,
stage2_bootstrap = stage2_bootstrap,
runtime_details = runtime_details,
venv = venv,
)
elif bootstrap_output:
_create_stage1_bootstrap(
Expand All @@ -282,6 +296,7 @@ def _create_executable(
is_for_zip = False,
imports = imports,
main_py = main_py,
venv = venv,
)
else:
# Otherwise, this should be the Windows case of launcher + zip.
Expand All @@ -296,13 +311,20 @@ def _create_executable(
build_zip_enabled = build_zip_enabled,
))

# The interpreter is added this late in the process so that it isn't
# added to the zipped files.
if venv:
extra_runfiles = extra_runfiles.merge(ctx.runfiles([venv.interpreter]))
return create_executable_result_struct(
extra_files_to_build = depset(extra_files_to_build),
output_groups = {"python_zip_file": depset([zip_file])},
extra_runfiles = extra_runfiles,
)

def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details):
def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details, venv):
python_binary = _runfiles_root_path(ctx, venv.interpreter.short_path)
python_binary_actual = _runfiles_root_path(ctx, venv.interpreter_actual_path)

# The location of this file doesn't really matter. It's added to
# the zip file as the top-level __main__.py file and not included
# elsewhere.
Expand All @@ -311,7 +333,8 @@ def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details):
template = runtime_details.effective_runtime.zip_main_template,
output = output,
substitutions = {
"%python_binary%": runtime_details.executable_interpreter_path,
"%python_binary%": python_binary,
"%python_binary_actual%": python_binary_actual,
"%stage2_bootstrap%": "{}/{}".format(
ctx.workspace_name,
stage2_bootstrap.short_path,
Expand All @@ -321,6 +344,82 @@ def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details):
)
return output

# Create a venv the executable can use.
# For venv details and the venv startup process, see:
# * https://docs.python.org/3/library/venv.html
# * https://snarky.ca/how-virtual-environments-work/
# * https://github.com/python/cpython/blob/main/Modules/getpath.py
# * https://github.com/python/cpython/blob/main/Lib/site.py
def _create_venv(ctx, output_prefix, imports, runtime_details):
venv = "_{}.venv".format(output_prefix.lstrip("_"))

# The pyvenv.cfg file must be present to trigger the venv site hooks.
# Because it's paths are expected to be absolute paths, we can't reliably
# put much in it. See https://github.com/python/cpython/issues/83650
pyvenv_cfg = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv))
ctx.actions.write(pyvenv_cfg, "")

runtime = runtime_details.effective_runtime
if runtime.interpreter:
py_exe_basename = paths.basename(runtime.interpreter.short_path)

# Even though ctx.actions.symlink() is used, using
# declare_symlink() is required to ensure that the resulting file
# in runfiles is always a symlink. An RBE implementation, for example,
# may choose to write what symlink() points to instead.
interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename))
interpreter_actual_path = runtime.interpreter.short_path
parent = "/".join([".."] * (interpreter_actual_path.count("/") + 1))
rel_path = parent + "/" + interpreter_actual_path
ctx.actions.symlink(output = interpreter, target_path = rel_path)
else:
py_exe_basename = paths.basename(runtime.interpreter_path)
interpreter = ctx.actions.declare_symlink("{}/bin/{}".format(venv, py_exe_basename))
ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path)
interpreter_actual_path = runtime.interpreter_path

if runtime.interpreter_version_info:
version = "{}.{}".format(
runtime.interpreter_version_info.major,
runtime.interpreter_version_info.minor,
)
else:
version_flag = ctx.attr._python_version_flag[config_common.FeatureFlagInfo].value
version_flag_parts = version_flag.split(".")[0:2]
version = "{}.{}".format(*version_flag_parts)

# See site.py logic: free-threaded builds append "t" to the venv lib dir name
if "t" in runtime.abi_flags:
version += "t"

site_packages = "{}/lib/python{}/site-packages".format(venv, version)
pth = ctx.actions.declare_file("{}/bazel.pth".format(site_packages))
ctx.actions.write(pth, "import _bazel_site_init\n")

site_init = ctx.actions.declare_file("{}/_bazel_site_init.py".format(site_packages))
computed_subs = ctx.actions.template_dict()
computed_subs.add_joined("%imports%", imports, join_with = ":", map_each = _map_each_identity)
ctx.actions.expand_template(
template = runtime.site_init_template,
output = site_init,
substitutions = {
"%import_all%": "True" if ctx.fragments.bazel_py.python_import_all_repositories else "False",
"%site_init_runfiles_path%": "{}/{}".format(ctx.workspace_name, site_init.short_path),
"%workspace_name%": ctx.workspace_name,
},
computed_substitutions = computed_subs,
)

return struct(
interpreter = interpreter,
# Runfiles-relative path or absolute path
interpreter_actual_path = interpreter_actual_path,
files_without_interpreter = [pyvenv_cfg, pth, site_init],
)

def _map_each_identity(v):
return v

def _create_stage2_bootstrap(
ctx,
*,
Expand Down Expand Up @@ -363,6 +462,13 @@ def _create_stage2_bootstrap(
)
return output

def _runfiles_root_path(ctx, path):
# The ../ comes from short_path for files in other repos.
if path.startswith("../"):
return path[3:]
else:
return "{}/{}".format(ctx.workspace_name, path)

def _create_stage1_bootstrap(
ctx,
*,
Expand All @@ -371,12 +477,24 @@ def _create_stage1_bootstrap(
stage2_bootstrap = None,
imports = None,
is_for_zip,
runtime_details):
runtime_details,
venv = None):
runtime = runtime_details.effective_runtime

if venv:
python_binary_path = _runfiles_root_path(ctx, venv.interpreter.short_path)
else:
python_binary_path = runtime_details.executable_interpreter_path

if is_for_zip and venv:
python_binary_actual = _runfiles_root_path(ctx, venv.interpreter_actual_path)
else:
python_binary_actual = ""

subs = {
"%is_zipfile%": "1" if is_for_zip else "0",
"%python_binary%": runtime_details.executable_interpreter_path,
"%python_binary%": python_binary_path,
"%python_binary_actual%": python_binary_actual,
"%target%": str(ctx.label),
"%workspace_name%": ctx.workspace_name,
}
Expand Down Expand Up @@ -447,6 +565,7 @@ def _create_windows_exe_launcher(
)

def _create_zip_file(ctx, *, output, original_nonzip_executable, zip_main, runfiles):
"""Create a Python zipapp (zip with __main__.py entry point)."""
workspace_name = ctx.workspace_name
legacy_external_runfiles = _py_builtins.get_legacy_external_runfiles(ctx)

Expand Down Expand Up @@ -524,7 +643,14 @@ def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles):
zip_runfiles_path = paths.normalize("{}/{}".format(workspace_name, path))
return "{}/{}".format(_ZIP_RUNFILES_DIRECTORY_NAME, zip_runfiles_path)

def _create_executable_zip_file(ctx, *, output, zip_file, stage2_bootstrap, runtime_details):
def _create_executable_zip_file(
ctx,
*,
output,
zip_file,
stage2_bootstrap,
runtime_details,
venv):
prelude = ctx.actions.declare_file(
"{}_zip_prelude.sh".format(output.basename),
sibling = output,
Expand All @@ -536,6 +662,7 @@ def _create_executable_zip_file(ctx, *, output, zip_file, stage2_bootstrap, runt
stage2_bootstrap = stage2_bootstrap,
runtime_details = runtime_details,
is_for_zip = True,
venv = venv,
)
else:
ctx.actions.write(prelude, "#!/usr/bin/env python3\n")
Expand Down
Loading

0 comments on commit 8ff4386

Please sign in to comment.