diff --git a/bootstrap.egg-info/entry_points.txt b/bootstrap.egg-info/entry_points.txt index c00d1d3a02..a21ca22709 100644 --- a/bootstrap.egg-info/entry_points.txt +++ b/bootstrap.egg-info/entry_points.txt @@ -2,6 +2,7 @@ egg_info = setuptools.command.egg_info:egg_info build_py = setuptools.command.build_py:build_py sdist = setuptools.command.sdist:sdist +editable_wheel = setuptools.command.editable_wheel:editable_wheel [distutils.setup_keywords] include_package_data = setuptools.dist:assert_bool diff --git a/changelog.d/3265.change.rst b/changelog.d/3265.change.rst new file mode 100644 index 0000000000..a21621e979 --- /dev/null +++ b/changelog.d/3265.change.rst @@ -0,0 +1,17 @@ +Added implementation for *editable install* hooks (PEP 660) - **beta** stage. + +- The user will be able select between two distinct behaviors: + + - *lenient*, which prioritises the ability of the users of changing the + distributed packages (e.g. adding new files or removing old ones) + + - *strict*, which will try to replicate as much as possible the behavior of + the package as if it would be normally installed by end users. + The *strict* editable installation is not able to detect if files are + added or removed from the project (a new installation is required). + +.. important:: + The *editable* aspect of the *editable install* supported this implementation + is restricted to the Python modules contained in the distributed package. + Changes in binary extensions (e.g. C/C++), entry-point definitions, + dependencies, metadata, datafiles, etc require a new installation. diff --git a/changelog.d/3380.change.rst b/changelog.d/3380.change.rst new file mode 100644 index 0000000000..9622417a07 --- /dev/null +++ b/changelog.d/3380.change.rst @@ -0,0 +1,10 @@ +Improved the handling of the ``config_settings`` parameter in both PEP 517 and +PEP 660 interfaces: + +- It is possible now to pass both ``--global-option`` and ``--build-option``. + As discussed in #1928, arbitrary arguments passed via ``--global-option`` + should be placed before the name of the setuptools' internal command, while + ``--build-option`` should come after. + +- Users can pass ``editable-mode=strict`` to select a strict behaviour for the + editable installation. diff --git a/changelog.d/3380.deprecation.rst b/changelog.d/3380.deprecation.rst new file mode 100644 index 0000000000..54d3c4c37a --- /dev/null +++ b/changelog.d/3380.deprecation.rst @@ -0,0 +1,7 @@ +Passing some types of parameters via ``--global-option`` to setuptools PEP 517/PEP 660 backend +is now considered deprecated. The user can pass the same arbitrary parameter +via ``--build-option`` (``--global-option`` is now reserved for flags like +``--verbose`` or ``--quiet``). + +Both ``--build-option`` and ``--global-option`` are supported as a **transitional** effort (a.k.a. "escape hatch"). +In the future a proper list of allowed ``config_settings`` may be created. diff --git a/changelog.d/3392.change.rst b/changelog.d/3392.change.rst new file mode 100644 index 0000000000..8ae7fd90de --- /dev/null +++ b/changelog.d/3392.change.rst @@ -0,0 +1,5 @@ +Exposed ``get_output_mapping()`` from ``build_py`` and ``build_ext`` +subcommands. This interface is reserved for the use of ``setuptools`` +Extensions and third part packages are explicitly disallowed to calling it. +However, any implementation overwriting ``build_py`` or ``build_ext`` are +required to honour this interface. diff --git a/changelog.d/3412.change.rst b/changelog.d/3412.change.rst new file mode 100644 index 0000000000..69f02bc814 --- /dev/null +++ b/changelog.d/3412.change.rst @@ -0,0 +1,3 @@ +Added ability of collecting source files from custom build sub-commands to +``sdist``. This allows plugins and customization scripts to automatically +add required source files in the source distribution. diff --git a/changelog.d/3414.change.rst b/changelog.d/3414.change.rst new file mode 100644 index 0000000000..b29f2c5512 --- /dev/null +++ b/changelog.d/3414.change.rst @@ -0,0 +1,4 @@ +Users can *temporarily* specify an environment variable +``SETUPTOOLS_ENABLE_FEATURE=legacy-editable`` as a escape hatch for the +:pep:`660` behavior. This setting is **transitional** and may be removed in the +future. diff --git a/changelog.d/3414.doc.rst b/changelog.d/3414.doc.rst new file mode 100644 index 0000000000..b4756dae64 --- /dev/null +++ b/changelog.d/3414.doc.rst @@ -0,0 +1,2 @@ +Updated :doc:`Development Mode ` to reflect on the +implementation of :pep:`660`. diff --git a/changelog.d/3484.change.rst b/changelog.d/3484.change.rst new file mode 100644 index 0000000000..3c671cb8c9 --- /dev/null +++ b/changelog.d/3484.change.rst @@ -0,0 +1,6 @@ +Added *transient* ``compat`` mode to editable installs. +This more will be temporarily available (to facilitate the transition period) +for those that want to emulate the behavior of the ``develop`` command +(in terms of what is added to ``sys.path``). +This mode is provided "as is", with limited support, and will be removed in +future versions of ``setuptools``. diff --git a/docs/userguide/development_mode.rst b/docs/userguide/development_mode.rst index 4130ab7329..6c2d212a59 100644 --- a/docs/userguide/development_mode.rst +++ b/docs/userguide/development_mode.rst @@ -1,34 +1,259 @@ -Development Mode -================ - -Under normal circumstances, the ``setuptools`` assume that you are going to -build a distribution of your project, not use it in its "raw" or "unbuilt" -form. However, if you were to use the ``setuptools`` to build a distribution, -you would have to rebuild and reinstall your project every time you made a -change to it during development. - -Another problem that sometimes comes is that you may -need to do development on two related projects at the same time. You may need -to put both projects' packages in the same directory to run them, but need to -keep them separate for revision control purposes. How can you do this? - -Setuptools allows you to deploy your projects for use in a common directory or -staging area, but without copying any files. Thus, you can edit each project's -code in its checkout directory, and only need to run build commands when you -change files that need to be compiled or the provided metadata and setuptools configuration. - -You can perform a ``pip`` installation passing the ``-e/--editable`` -flag (e.g., ``pip install -e .``). It works very similarly to -``pip install .``, except that it doesn't actually install anything. -Instead, it creates a special ``.egg-link`` file in the target directory -(usually ``site-packages``) that links to your project's source code. -It may also update an existing ``easy-install.pth`` file -to include your project's source code, thereby making -it available on ``sys.path`` for all programs using that Python installation. - -You can deploy the same project to multiple staging areas, e.g., if you have -multiple projects on the same machine that are sharing the same project you're -doing development work. +Development Mode (a.k.a. "Editable Installs") +============================================= + +When creating a Python project, developers usually want to implement and test +changes iteratively, before cutting a release and preparing a distribution archive. + +In normal circumstances this can be quite cumbersome and require the developers +to manipulate the ``PATHONPATH`` environment variable or to continuous re-build +and re-install the project. + +To facilitate iterative exploration and experimentation, setuptools allows +users to instruct the Python interpreter and its import machinery to load the +code under development directly from the project folder without having to +copy the files to a different location in the disk. +This means that changes in the Python source code can immediately take place +without requiring a new installation. + +You can enter this "development mode" by performing an :doc:`editable installation +` inside of a :term:`virtual environment`, +using :doc:`pip's ` ``-e/--editable`` flag, as shown bellow: + +.. code-block:: bash + + $ cd your-python-project + $ python -m venv .venv + # Activate your environemt with: + # `source .venv/bin/activate` on Unix/macOS + # or `.venv\Scripts\activate` on Windows + + $ pip install --editable . + + # Now you have access to your package + # as if it was installed in .venv + $ python -c "import your_python_project" + + +An "editable installation" works very similarly to a regular install with +``pip install .``, except that it only installs your package dependencies, +metadata and wrappers for :ref:`console and GUI scripts `. +Under the hood, setuptools will try to create a special :mod:`.pth file ` +in the target directory (usually ``site-packages``) that extends the +``PYTHONPATH`` or install a custom :doc:`import hook `. When you're done with a given development task, you can simply uninstall your package (as you would normally do with ``pip uninstall ``). + +Please note that, by default an editable install will expose at least all the +files that would be available in a regular installation. However, depending on +the file and directory organization in your project, it might also expose +as a side effect files that would not be normally available. +This is allowed so you can iteratively create new Python modules. +Please have a look on the following section if you are looking for a different behaviour. + +.. admonition:: Virtual Environments + + You can think about virtual environments as "isolated Python runtime deployments" + that allow users to install different sets of libraries and tools without + messing with the global behaviour of the system. + + They are a safe way of testing new projects and can be created easily + with the :mod:`venv` module from the standard library. + + Please note however that depending on your operating system or distribution, + ``venv`` might not come installed by default with Python. For those cases, + you might need to use the OS package manager to install it. + For example, in Debian/Ubuntu-based systems you can obtain it via: + + .. code-block:: bash + + sudo apt install python3-venv + + Alternatively, you can also try installing :pypi:`virtualenᴠ`. + More information is available on the Python Packaging User Guide on + :doc:`PyPUG:guides/installing-using-pip-and-virtual-environments`. + +.. note:: + .. versionchanged:: v64.0.0 + Editable installation hooks implemented according to :pep:`660`. + Support for :pep:`namespace packages <420>` is still **EXPERIMENTAL**. + + +"Strict" editable installs +-------------------------- + +When thinking about editable installations, users might have the following +expectations: + +1. It should allow developers to add new files (or split/rename existing ones) + and have them automatically exposed. +2. It should behave as close as possible to a regular installation and help + users to detect problems (e.g. new files not being included in the distribution). + +Unfortunately these expectations are in conflict with each other. +To solve this problem ``setuptools`` allows developers to choose a more +*"strict"* mode for the editable installation. This can be done by passing +a special *configuration setting* via :pypi:`pip`, as indicated bellow: + +.. code-block:: bash + + pip install -e . --config-settings editable_mode=strict + +In this mode, new files **won't** be exposed and the editable installs will +try to mimic as much as possible the behavior of a regular install. +Under the hood, ``setuptools`` will create a tree of file links in an auxiliary +directory (``$your_project_dir/build``) and add it to ``PYTHONPATH`` via a +:mod:`.pth file `. (Please be careful to not delete this repository +by mistake otherwise your files may stop being accessible). + +.. warning:: + Strict editable installs require auxiliary files to be placed in a + ``build/__editable__.*`` directory (relative to your project root). + + Please be careful to not remove this directory while testing your project, + otherwise your editable installation may be compromised. + + You can remove the ``build/__editable__.*`` directory after uninstalling. + + +.. note:: + .. versionadded:: v64.0.0 + *Strict* mode implemented as **EXPERIMENTAL**. + + +Limitations +----------- + +- The *editable* term is used to refer only to Python modules + inside the package directories. Non-Python files, external (data) files, + executable script files, binary extensions, headers and metadata may be + exposed as a *snapshot* of the version they were at the moment of the + installation. +- Adding new dependencies, entry-points or changing your project's metadata + require a fresh "editable" re-installation. +- Console scripts and GUI scripts **MUST** be specified via :doc:`entry-points + ` to work properly. +- *Strict* editable installs require the file system to support + either :wiki:`symbolic ` or :wiki:`hard links `. + This installation mode might also generate auxiliary files under the project directory. +- There is *no guarantee* that the editable installation will be performed + using a specific technique. Depending on each project, ``setuptools`` may + select a different approach to ensure the package is importable at runtime. +- There is *no guarantee* that files outside the top-level package directory + will be accessible after an editable install. +- There is *no guarantee* that attributes like ``__path__`` or ``__file__`` + will correspond to the exact location of the original files (e.g., + ``setuptools`` might employ file links to perform the editable installation). + Users are encouraged to use tools like :mod:`importlib.resources` or + :mod:`importlib.metadata` when trying to access package files directly. +- Editable installations may not work with + :doc:`namespaces created with pkgutil or pkg_resouces + `. + Please use :pep:`420`-style implicit namespaces [#namespaces]_. +- Support for :pep:`420`-style implicit namespace packages for + projects structured using :ref:`flat-layout` is still **experimental**. + If you experience problems, you can try converting your package structure + to the :ref:`src-layout`. + +.. attention:: + Editable installs are **not a perfect replacement for regular installs** + in a test environment. When in doubt, please test your projects as + installed via a regular wheel. There are tools in the Python ecosystem, + like :pypi:`tox` or :pypi:`nox`, that can help you with that + (when used with appropriate configuration). + + +Legacy Behavior +--------------- + +If your project is not compatible with the new "editable installs" or you wish +to replicate the legacy behavior, for the time being you can also perform the +installation in the ``compat`` mode: + +.. code-block:: bash + + pip install -e . --config-settings editable_mode=compat + +This installation mode will try to emulate how ``python setup.py develop`` +works (still within the context of :pep:`660`). + +.. warning:: + The ``compat`` mode is *transitional* and will be removed in + future versions of ``setuptools``, it exists only to help during the + migration period. + Also note that support for this mode is limited: + it is safe to assume that the ``compat`` mode is offered "as is", and + improvements are unlikely to be implemented. + Users are encouraged to try out the new editable installation techniques + and make the necessary adaptations. + +If the ``compat`` mode does not work for you, you can also disable the +:pep:`editable install <660>` hooks in ``setuptools`` by setting an environment +variable: + +.. code-block:: + + SETUPTOOLS_USE_FEATURE="legacy-editable" + +This *may* cause the installer (e.g. ``pip``) to effectively run the "legacy" +installation command: ``python setup.py develop`` [#installer]_. + + +How editable installations work? +-------------------------------- + +*Advanced topic* + +There are many techniques that can be used to expose packages under development +in such a way that they are available as if they were installed. +Depending on the project file structure and the selected mode, ``setuptools`` +will choose one of these approaches for the editable installation [#criteria]_. + +A non-exhaustive list of implementation mechanisms is presented below. +More information is available on the text of :pep:`PEP 660 <660#what-to-put-in-the-wheel>`. + +- A static ``.pth`` file [#static_pth]_ can be added to one of the directories + listed in :func:`site.getsitepackages` or :func:`site.getusersitepackages` to + extend :obj:`sys.path`. +- A directory containing a *farm of file links* that mimic the + project structure and point to the original files can be employed. + This directory can then be added to :obj:`sys.path` using a static ``.pth`` file. +- A dynamic ``.pth`` file [#dynamic_pth]_ can also be used to install an + "import :term:`finder`" (:obj:`~importlib.abc.MetaPathFinder` or + :obj:`~importlib.abc.PathEntryFinder`) that will hook into Python's + :doc:`import system ` machinery. + +.. attention:: + ``Setuptools`` offers **no guarantee** of which technique will be used to + perform an editable installation. This will vary from project to project + and may change depending on the specific version of ``setuptools`` being + used. + + +---- + +.. rubric:: Notes + +.. [#namespaces] + You *may* be able to use *strict* editable installations with namespace + packages created with ``pkgutil`` or ``pkg_namespaces``, however this is not + officially supported. + +.. [#installer] + For this workaround to work, the installer tool needs to support legacy + editable installations. (Future versions of ``pip``, for example, may drop + support for this feature). + +.. [#criteria] + ``setuptools`` strives to find a balance between allowing the user to see + the effects of project files being edited while still trying to keep the + editable installation as similar as possible to a regular installation. + +.. [#static_pth] + i.e., a ``.pth`` file where each line correspond to a path that should be + added to :obj:`sys.path`. See :mod:`Site-specific configuration hook `. + +.. [#dynamic_pth] + i.e., a ``.pth`` file that starts where each line starts with an ``import`` + statement and executes arbitrary Python code. See :mod:`Site-specific + configuration hook `. diff --git a/docs/userguide/entry_point.rst b/docs/userguide/entry_point.rst index eff20cf090..6ba00287d7 100644 --- a/docs/userguide/entry_point.rst +++ b/docs/userguide/entry_point.rst @@ -21,6 +21,8 @@ highlighting tool :pypi:`pygments` allows specifying additional styles using the entry point ``pygments.styles``. +.. _console-scripts: + Console Scripts =============== diff --git a/docs/userguide/extension.rst b/docs/userguide/extension.rst index 31d32d6174..b49816b007 100644 --- a/docs/userguide/extension.rst +++ b/docs/userguide/extension.rst @@ -56,8 +56,8 @@ a ``foo`` command, you might add something like this to your project: distutils.commands = foo = mypackage.some_module:foo -(Assuming, of course, that the ``foo`` class in ``mypackage.some_module`` is -a ``setuptools.Command`` subclass.) +Assuming, of course, that the ``foo`` class in ``mypackage.some_module`` is +a ``setuptools.Command`` subclass (documented bellow). Once a project containing such entry points has been activated on ``sys.path``, (e.g. by running ``pip install``) the command(s) will be available to any @@ -72,9 +72,22 @@ Custom commands should try to replicate the same overall behavior as the original classes, and when possible, even inherit from them. You should also consider handling exceptions such as ``CompileError``, -``LinkError``, ``LibError``, among others. These exceptions are available in +``LinkError``, ``LibError``, among others. These exceptions are available in the ``setuptools.errors`` module. +.. autoclass:: setuptools.Command + :members: + + +Supporting sdists and editable installs in ``build`` sub-commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``build`` sub-commands (like ``build_py`` and ``build_ext``) +are encouraged to implement the following protocol: + +.. autoclass:: setuptools.command.build.SubCommand + :members: + Adding Arguments ---------------- diff --git a/docs/userguide/pyproject_config.rst b/docs/userguide/pyproject_config.rst index 709bf91946..b1d4a4e3fe 100644 --- a/docs/userguide/pyproject_config.rst +++ b/docs/userguide/pyproject_config.rst @@ -7,10 +7,10 @@ Configuring setuptools using ``pyproject.toml`` files .. note:: New in 61.0.0 .. important:: - For the time being [#pep660-status]_, ``pip`` still might require a ``setup.py`` file - to support :doc:`editable installs ` [#setupcfg-caveats]_. - - A simple script will suffice, for example: + If compatibility with legacy builds or versions of tools that don't support + certain packaging standards (e.g. :pep:`517` or :pep:`660`), a simple ``setup.py`` + script can be added to your project [#setupcfg-caveats]_ + (while keeping the configuration in ``pyproject.toml``): .. code-block:: python @@ -214,12 +214,10 @@ however please keep in mind that all non-comment lines must conform with :pep:`5 .. rubric:: Notes -.. [#pep660-status] Editable install without ``setup.py`` will be supported in - future versions of ``setuptools``. Check https://github.com/pypa/setuptools/issues/2816 for detail. - .. [#setupcfg-caveats] ``pip`` may allow editable install only with ``pyproject.toml`` - and ``setup.cfg``. However, this behavior may not be consistent over various build - tools. Having a ``setup.py`` is still recommended if you rely on one of these tools. + and ``setup.cfg``. However, this behavior may not be consistent over various ``pip`` + versions and other packaging-related tools + (``setup.py`` is more reliable on those scenarios). .. [#entry-points] Dynamic ``scripts`` and ``gui-scripts`` are a special case. When resolving these metadata keys, ``setuptools`` will look for diff --git a/docs/userguide/quickstart.rst b/docs/userguide/quickstart.rst index 24c71b8e52..bf76f2c899 100644 --- a/docs/userguide/quickstart.rst +++ b/docs/userguide/quickstart.rst @@ -117,8 +117,7 @@ distributing into something that looks like the following (optional files marked with ``#``):: mypackage - ├── pyproject.toml - | # setup.cfg or setup.py (depending on the configuration method) + ├── pyproject.toml # and/or setup.cfg/setup.py (depending on the configuration method) | # README.rst or README.md (a nice description of your package) | # LICENCE (properly chosen license information, e.g. MIT, BSD-3, GPL-3, MPL-2, etc...) └── mypackage @@ -378,19 +377,18 @@ Here's how to do it:: pip install --editable . -This creates a link file in your interpreter site package directory which -associate with your source code. For more information, see :doc:`development_mode`. +See :doc:`development_mode` for more information. .. tip:: Prior to :ref:`pip v21.1 `, a ``setup.py`` script was required to be compatible with development mode. With late - versions of pip, ``setup.cfg``-only projects may be installed in this mode. + versions of pip, projects without ``setup.py`` may be installed in this mode. - If you are experimenting with :doc:`configuration using pyproject.toml `, - or have version of ``pip`` older than v21.1, you might need to keep a + If you have a version of ``pip`` older than v21.1 or is using a different + packaging-related tool that does not support :pep:`660`, you might need to keep a ``setup.py`` file in file in your repository if you want to use editable - installs (for the time being). + installs. A simple script will suffice, for example: @@ -400,8 +398,9 @@ associate with your source code. For more information, see :doc:`development_mod setup() - You can still keep all the configuration in :doc:`setup.cfg ` - (or :doc:`pyproject.toml `). + You can still keep all the configuration in + :doc:`pyproject.toml ` and/or + :doc:`setup.cfg ` Uploading your package to PyPI diff --git a/pytest.ini b/pytest.ini index 7c863960b1..aed8b7168c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -60,3 +60,4 @@ filterwarnings= ignore:Setuptools is replacing distutils ignore:Support for .* in .pyproject.toml. is still .beta. + ignore::setuptools.command.editable_wheel.InformationOnly diff --git a/setup.cfg b/setup.cfg index ce714fb18b..e5754f484c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -117,6 +117,7 @@ distutils.commands = develop = setuptools.command.develop:develop dist_info = setuptools.command.dist_info:dist_info easy_install = setuptools.command.easy_install:easy_install + editable_wheel = setuptools.command.editable_wheel:editable_wheel egg_info = setuptools.command.egg_info:egg_info install = setuptools.command.install:install install_egg_info = setuptools.command.install_egg_info:install_egg_info diff --git a/setuptools/__init__.py b/setuptools/__init__.py index cff04323e9..6c24cc2b30 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -94,7 +94,59 @@ def setup(**attrs): class Command(_Command): - __doc__ = _Command.__doc__ + """ + Setuptools internal actions are organized using a *command design pattern*. + This means that each action (or group of closely related actions) executed during + the build should be implemented as a ``Command`` subclass. + + These commands are abstractions and do not necessarily correspond to a command that + can (or should) be executed via a terminal, in a CLI fashion (although historically + they would). + + When creating a new command from scratch, custom defined classes **SHOULD** inherit + from ``setuptools.Command`` and implement a few mandatory methods. + Between these mandatory methods, are listed: + + .. method:: initialize_options(self) + + Set or (reset) all options/attributes/caches used by the command + to their default values. Note that these values may be overwritten during + the build. + + .. method:: finalize_options(self) + + Set final values for all options/attributes used by the command. + Most of the time, each option/attribute/cache should only be set if it does not + have any value yet (e.g. ``if self.attr is None: self.attr = val``). + + .. method:: run(self) + + Execute the actions intended by the command. + (Side effects **SHOULD** only take place when ``run`` is executed, + for example, creating new files or writing to the terminal output). + + A useful analogy for command classes is to think of them as subroutines with local + variables called "options". The options are "declared" in ``initialize_options()`` + and "defined" (given their final values, aka "finalized") in ``finalize_options()``, + both of which must be defined by every command class. The "body" of the subroutine, + (where it does all the work) is the ``run()`` method. + Between ``initialize_options()`` and ``finalize_options()``, ``setuptools`` may set + the values for options/attributes based on user's input (or circumstance), + which means that the implementation should be careful to not overwrite values in + ``finalize_options`` unless necessary. + + Please note that other commands (or other parts of setuptools) may also overwrite + the values of the command's options/attributes multiple times during the build + process. + Therefore it is important to consistently implement ``initialize_options()`` and + ``finalize_options()``. For example, all derived attributes (or attributes that + depend on the value of other attributes) **SHOULD** be recomputed in + ``finalize_options``. + + When overwriting existing commands, custom defined classes **MUST** abide by the + same APIs implemented by the original class. They also **SHOULD** inherit from the + original class. + """ command_consumes_arguments = False @@ -122,6 +174,12 @@ def ensure_string_list(self, option): currently a string, we split it either on /,\s*/ or /\s+/, so "foo bar baz", "foo,bar,baz", and "foo, bar baz" all become ["foo", "bar", "baz"]. + + .. + TODO: This method seems to be similar to the one in ``distutils.cmd`` + Probably it is just here for backward compatibility with old Python versions? + + :meta private: """ val = getattr(self, option) if val is None: diff --git a/setuptools/_path.py b/setuptools/_path.py index ede9cb0027..3767523b78 100644 --- a/setuptools/_path.py +++ b/setuptools/_path.py @@ -1,7 +1,29 @@ import os +from typing import Union + +_Path = Union[str, os.PathLike] def ensure_directory(path): """Ensure that the parent directory of `path` exists""" dirname = os.path.dirname(path) os.makedirs(dirname, exist_ok=True) + + +def same_path(p1: _Path, p2: _Path) -> bool: + """Differs from os.path.samefile because it does not require paths to exist. + Purely string based (no comparison between i-nodes). + >>> same_path("a/b", "./a/b") + True + >>> same_path("a/b", "a/./b") + True + >>> same_path("a/b", "././a/b") + True + >>> same_path("a/b", "./a/b/c/..") + True + >>> same_path("a/b", "../a/b/c") + False + >>> same_path("a", "a/b") + False + """ + return os.path.normpath(p1) == os.path.normpath(p2) diff --git a/setuptools/build_meta.py b/setuptools/build_meta.py index a0d46a7989..98fb148a5d 100644 --- a/setuptools/build_meta.py +++ b/setuptools/build_meta.py @@ -28,17 +28,22 @@ import io import os +import shlex import sys import tokenize import shutil import contextlib import tempfile import warnings +from pathlib import Path +from typing import Dict, Iterator, List, Optional, Union import setuptools import distutils +from ._path import same_path from ._reqs import parse_strings -from .extern.more_itertools import always_iterable +from ._deprecation_warning import SetuptoolsDeprecationWarning +from distutils.util import strtobool __all__ = ['get_requires_for_build_sdist', @@ -46,9 +51,15 @@ 'prepare_metadata_for_build_wheel', 'build_wheel', 'build_sdist', + 'get_requires_for_build_editable', + 'prepare_metadata_for_build_editable', + 'build_editable', '__legacy__', 'SetupRequirementsError'] +SETUPTOOLS_ENABLE_FEATURES = os.getenv("SETUPTOOLS_ENABLE_FEATURES", "").lower() +LEGACY_EDITABLE = "legacy-editable" in SETUPTOOLS_ENABLE_FEATURES.replace("_", "-") + class SetupRequirementsError(BaseException): def __init__(self, specifiers): @@ -127,33 +138,182 @@ def suppress_known_deprecation(): yield -class _BuildMetaBackend: +_ConfigSettings = Optional[Dict[str, Union[str, List[str], None]]] +""" +Currently the user can run:: + + pip install -e . --config-settings key=value + python -m build -C--key=value -C key=value + +- pip will pass both key and value as strings and overwriting repeated keys + (pypa/pip#11059). +- build will accumulate values associated with repeated keys in a list. + It will also accept keys with no associated value. + This means that an option passed by build can be ``str | list[str] | None``. +- PEP 517 specifies that ``config_settings`` is an optional dict. +""" + + +class _ConfigSettingsTranslator: + """Translate ``config_settings`` into distutils-style command arguments. + Only a limited number of options is currently supported. + """ + # See pypa/setuptools#1928 pypa/setuptools#2491 - @staticmethod - def _fix_config(config_settings): + def _get_config(self, key: str, config_settings: _ConfigSettings) -> List[str]: """ - Ensure config settings meet certain expectations. - - >>> fc = _BuildMetaBackend._fix_config - >>> fc(None) - {'--global-option': []} - >>> fc({}) - {'--global-option': []} - >>> fc({'--global-option': 'foo'}) - {'--global-option': ['foo']} - >>> fc({'--global-option': ['foo']}) - {'--global-option': ['foo']} + Get the value of a specific key in ``config_settings`` as a list of strings. + + >>> fn = _ConfigSettingsTranslator()._get_config + >>> fn("--global-option", None) + [] + >>> fn("--global-option", {}) + [] + >>> fn("--global-option", {'--global-option': 'foo'}) + ['foo'] + >>> fn("--global-option", {'--global-option': ['foo']}) + ['foo'] + >>> fn("--global-option", {'--global-option': 'foo'}) + ['foo'] + >>> fn("--global-option", {'--global-option': 'foo bar'}) + ['foo', 'bar'] """ - config_settings = config_settings or {} - config_settings['--global-option'] = list(always_iterable( - config_settings.get('--global-option'))) - return config_settings + cfg = config_settings or {} + opts = cfg.get(key) or [] + return shlex.split(opts) if isinstance(opts, str) else opts + + def _valid_global_options(self): + """Global options accepted by setuptools (e.g. quiet or verbose).""" + options = (opt[:2] for opt in setuptools.dist.Distribution.global_options) + return {flag for long_and_short in options for flag in long_and_short if flag} + + def _global_args(self, config_settings: _ConfigSettings) -> Iterator[str]: + """ + Let the user specify ``verbose`` or ``quiet`` + escape hatch via + ``--global-option``. + Note: ``-v``, ``-vv``, ``-vvv`` have similar effects in setuptools, + so we just have to cover the basic scenario ``-v``. + + >>> fn = _ConfigSettingsTranslator()._global_args + >>> list(fn(None)) + [] + >>> list(fn({"verbose": "False"})) + ['-q'] + >>> list(fn({"verbose": "1"})) + ['-v'] + >>> list(fn({"--verbose": None})) + ['-v'] + >>> list(fn({"verbose": "true", "--global-option": "-q --no-user-cfg"})) + ['-v', '-q', '--no-user-cfg'] + >>> list(fn({"--quiet": None})) + ['-q'] + """ + cfg = config_settings or {} + falsey = {"false", "no", "0", "off"} + if "verbose" in cfg or "--verbose" in cfg: + level = str(cfg.get("verbose") or cfg.get("--verbose") or "1") + yield ("-q" if level.lower() in falsey else "-v") + if "quiet" in cfg or "--quiet" in cfg: + level = str(cfg.get("quiet") or cfg.get("--quiet") or "1") + yield ("-v" if level.lower() in falsey else "-q") + + valid = self._valid_global_options() + args = self._get_config("--global-option", config_settings) + yield from (arg for arg in args if arg.strip("-") in valid) + + def __dist_info_args(self, config_settings: _ConfigSettings) -> Iterator[str]: + """ + The ``dist_info`` command accepts ``tag-date`` and ``tag-build``. + + .. warning:: + We cannot use this yet as it requires the ``sdist`` and ``bdist_wheel`` + commands run in ``build_sdist`` and ``build_wheel`` to re-use the egg-info + directory created in ``prepare_metadata_for_build_wheel``. + + >>> fn = _ConfigSettingsTranslator()._ConfigSettingsTranslator__dist_info_args + >>> list(fn(None)) + [] + >>> list(fn({"tag-date": "False"})) + ['--no-date'] + >>> list(fn({"tag-date": None})) + ['--no-date'] + >>> list(fn({"tag-date": "true", "tag-build": ".a"})) + ['--tag-date', '--tag-build', '.a'] + """ + cfg = config_settings or {} + if "tag-date" in cfg: + val = strtobool(str(cfg["tag-date"] or "false")) + yield ("--tag-date" if val else "--no-date") + if "tag-build" in cfg: + yield from ["--tag-build", str(cfg["tag-build"])] + + def _editable_args(self, config_settings: _ConfigSettings) -> Iterator[str]: + """ + The ``editable_wheel`` command accepts ``editable-mode=strict``. + + >>> fn = _ConfigSettingsTranslator()._editable_args + >>> list(fn(None)) + [] + >>> list(fn({"editable-mode": "strict"})) + ['--mode', 'strict'] + """ + cfg = config_settings or {} + mode = cfg.get("editable-mode") or cfg.get("editable_mode") + if not mode: + return + yield from ["--mode", str(mode)] + + def _arbitrary_args(self, config_settings: _ConfigSettings) -> Iterator[str]: + """ + Users may expect to pass arbitrary lists of arguments to a command + via "--global-option" (example provided in PEP 517 of a "escape hatch"). + + >>> fn = _ConfigSettingsTranslator()._arbitrary_args + >>> list(fn(None)) + [] + >>> list(fn({})) + [] + >>> list(fn({'--build-option': 'foo'})) + ['foo'] + >>> list(fn({'--build-option': ['foo']})) + ['foo'] + >>> list(fn({'--build-option': 'foo'})) + ['foo'] + >>> list(fn({'--build-option': 'foo bar'})) + ['foo', 'bar'] + >>> warnings.simplefilter('error', SetuptoolsDeprecationWarning) + >>> list(fn({'--global-option': 'foo'})) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + SetuptoolsDeprecationWarning: ...arguments given via `--global-option`... + """ + args = self._get_config("--global-option", config_settings) + global_opts = self._valid_global_options() + bad_args = [] + + for arg in args: + if arg.strip("-") not in global_opts: + bad_args.append(arg) + yield arg + + yield from self._get_config("--build-option", config_settings) + + if bad_args: + msg = f""" + The arguments {bad_args!r} were given via `--global-option`. + Please use `--build-option` instead, + `--global-option` is reserved to flags like `--verbose` or `--quiet`. + """ + warnings.warn(msg, SetuptoolsDeprecationWarning) - def _get_build_requires(self, config_settings, requirements): - config_settings = self._fix_config(config_settings) - sys.argv = sys.argv[:1] + ['egg_info'] + \ - config_settings["--global-option"] +class _BuildMetaBackend(_ConfigSettingsTranslator): + def _get_build_requires(self, config_settings, requirements): + sys.argv = [ + *sys.argv[:1], + *self._global_args(config_settings), + "egg_info", + *self._arbitrary_args(config_settings), + ] try: with Distribution.patch(): self.run_setup() @@ -174,57 +334,57 @@ def run_setup(self, setup_script='setup.py'): exec(code, locals()) def get_requires_for_build_wheel(self, config_settings=None): - return self._get_build_requires( - config_settings, requirements=['wheel']) + return self._get_build_requires(config_settings, requirements=['wheel']) def get_requires_for_build_sdist(self, config_settings=None): return self._get_build_requires(config_settings, requirements=[]) - def prepare_metadata_for_build_wheel(self, metadata_directory, - config_settings=None): - sys.argv = sys.argv[:1] + [ - 'dist_info', '--egg-base', metadata_directory] - with no_install_setup_requires(): - self.run_setup() - - dist_info_directory = metadata_directory - while True: - dist_infos = [f for f in os.listdir(dist_info_directory) - if f.endswith('.dist-info')] + def _bubble_up_info_directory(self, metadata_directory: str, suffix: str) -> str: + """ + PEP 517 requires that the .dist-info directory be placed in the + metadata_directory. To comply, we MUST copy the directory to the root. - if ( - len(dist_infos) == 0 and - len(_get_immediate_subdirectories(dist_info_directory)) == 1 - ): + Returns the basename of the info directory, e.g. `proj-0.0.0.dist-info`. + """ + candidates = list(Path(metadata_directory).glob(f"**/*{suffix}/")) + assert len(candidates) == 1, f"Exactly one {suffix} should have been produced" + info_dir = candidates[0] - dist_info_directory = os.path.join( - dist_info_directory, os.listdir(dist_info_directory)[0]) - continue + if not same_path(info_dir.parent, metadata_directory): + shutil.move(str(info_dir), metadata_directory) + # PEP 517 allow other files and dirs to exist in metadata_directory - assert len(dist_infos) == 1 - break + return info_dir.name - # PEP 517 requires that the .dist-info directory be placed in the - # metadata_directory. To comply, we MUST copy the directory to the root - if dist_info_directory != metadata_directory: - shutil.move( - os.path.join(dist_info_directory, dist_infos[0]), - metadata_directory) - shutil.rmtree(dist_info_directory, ignore_errors=True) + def prepare_metadata_for_build_wheel(self, metadata_directory, + config_settings=None): + sys.argv = [ + *sys.argv[:1], + *self._global_args(config_settings), + "dist_info", + "--output-dir", metadata_directory, + "--keep-egg-info", + ] + with no_install_setup_requires(): + self.run_setup() - return dist_infos[0] + self._bubble_up_info_directory(metadata_directory, ".egg-info") + return self._bubble_up_info_directory(metadata_directory, ".dist-info") def _build_with_temp_dir(self, setup_command, result_extension, result_directory, config_settings): - config_settings = self._fix_config(config_settings) result_directory = os.path.abspath(result_directory) # Build in a temporary directory, then copy to the target. os.makedirs(result_directory, exist_ok=True) with tempfile.TemporaryDirectory(dir=result_directory) as tmp_dist_dir: - sys.argv = (sys.argv[:1] + setup_command + - ['--dist-dir', tmp_dist_dir] + - config_settings["--global-option"]) + sys.argv = [ + *sys.argv[:1], + *self._global_args(config_settings), + *setup_command, + "--dist-dir", tmp_dist_dir, + *self._arbitrary_args(config_settings), + ] with no_install_setup_requires(): self.run_setup() @@ -249,6 +409,39 @@ def build_sdist(self, sdist_directory, config_settings=None): '.tar.gz', sdist_directory, config_settings) + def _get_dist_info_dir(self, metadata_directory: Optional[str]) -> Optional[str]: + if not metadata_directory: + return None + dist_info_candidates = list(Path(metadata_directory).glob("*.dist-info")) + assert len(dist_info_candidates) <= 1 + return str(dist_info_candidates[0]) if dist_info_candidates else None + + if not LEGACY_EDITABLE: + + # PEP660 hooks: + # build_editable + # get_requires_for_build_editable + # prepare_metadata_for_build_editable + def build_editable( + self, wheel_directory, config_settings=None, metadata_directory=None + ): + # XXX can or should we hide our editable_wheel command normally? + info_dir = self._get_dist_info_dir(metadata_directory) + opts = ["--dist-info-dir", info_dir] if info_dir else [] + cmd = ["editable_wheel", *opts, *self._editable_args(config_settings)] + return self._build_with_temp_dir( + cmd, ".whl", wheel_directory, config_settings + ) + + def get_requires_for_build_editable(self, config_settings=None): + return self.get_requires_for_build_wheel(config_settings) + + def prepare_metadata_for_build_editable(self, metadata_directory, + config_settings=None): + return self.prepare_metadata_for_build_wheel( + metadata_directory, config_settings + ) + class _BuildMetaLegacyBackend(_BuildMetaBackend): """Compatibility backend for setuptools @@ -299,6 +492,11 @@ def run_setup(self, setup_script='setup.py'): build_wheel = _BACKEND.build_wheel build_sdist = _BACKEND.build_sdist +if not LEGACY_EDITABLE: + get_requires_for_build_editable = _BACKEND.get_requires_for_build_editable + prepare_metadata_for_build_editable = _BACKEND.prepare_metadata_for_build_editable + build_editable = _BACKEND.build_editable + # The legacy backend __legacy__ = _BuildMetaLegacyBackend() diff --git a/setuptools/command/build.py b/setuptools/command/build.py index 12a4362209..283999dadf 100644 --- a/setuptools/command/build.py +++ b/setuptools/command/build.py @@ -1,8 +1,17 @@ -from distutils.command.build import build as _build +import sys import warnings +from typing import TYPE_CHECKING, List, Dict +from distutils.command.build import build as _build from setuptools import SetuptoolsDeprecationWarning +if sys.version_info >= (3, 8): + from typing import Protocol +elif TYPE_CHECKING: + from typing_extensions import Protocol +else: + from abc import ABC as Protocol + _ORIGINAL_SUBCOMMANDS = {"build_py", "build_clib", "build_ext", "build_scripts"} @@ -22,3 +31,116 @@ def run(self): warnings.warn(msg, SetuptoolsDeprecationWarning) self.sub_commands = _build.sub_commands super().run() + + +class SubCommand(Protocol): + """In order to support editable installations (see :pep:`660`) all + build subcommands **SHOULD** implement this protocol. They also **MUST** inherit + from ``setuptools.Command``. + + When creating an :pep:`editable wheel <660>`, ``setuptools`` will try to evaluate + custom ``build`` subcommands using the following procedure: + + 1. ``setuptools`` will set the ``editable_mode`` attribute to ``True`` + 2. ``setuptools`` will execute the ``run()`` command. + + .. important:: + Subcommands **SHOULD** take advantage of ``editable_mode=True`` to adequate + its behaviour or perform optimisations. + + For example, if a subcommand don't need to generate any extra file and + everything it does is to copy a source file into the build directory, + ``run()`` **SHOULD** simply "early return". + + Similarly, if the subcommand creates files that would be placed alongside + Python files in the final distribution, during an editable install + the command **SHOULD** generate these files "in place" (i.e. write them to + the original source directory, instead of using the build directory). + Note that ``get_output_mapping()`` should reflect that and include mappings + for "in place" builds accordingly. + + 3. ``setuptools`` use any knowledge it can derive from the return values of + ``get_outputs()`` and ``get_output_mapping()`` to create an editable wheel. + When relevant ``setuptools`` **MAY** attempt to use file links based on the value + of ``get_output_mapping()``. Alternatively, ``setuptools`` **MAY** attempt to use + :doc:`import hooks ` to redirect any attempt to import + to the directory with the original source code and other files built in place. + + Please note that custom sub-commands **SHOULD NOT** rely on ``run()`` being + executed (or not) to provide correct return values for ``get_outputs()``, + ``get_output_mapping()`` or ``get_source_files()``. The ``get_*`` methods should + work independently of ``run()``. + """ + + editable_mode: bool = False + """Boolean flag that will be set to ``True`` when setuptools is used for an + editable installation (see :pep:`660`). + Implementations **SHOULD** explicitly set the default value of this attribute to + ``False``. + When subcommands run, they can use this flag to perform optimizations or change + their behaviour accordingly. + """ + + build_lib: str + """String representing the directory where the build artifacts should be stored, + e.g. ``build/lib``. + For example, if a distribution wants to provide a Python module named ``pkg.mod``, + then a corresponding file should be written to ``{build_lib}/package/module.py``. + A way of thinking about this is that the files saved under ``build_lib`` + would be eventually copied to one of the directories in :obj:`site.PREFIXES` + upon installation. + + A command that produces platform-independent files (e.g. compiling text templates + into Python functions), **CAN** initialize ``build_lib`` by copying its value from + the ``build_py`` command. On the other hand, a command that produces + platform-specific files **CAN** initialize ``build_lib`` by copying its value from + the ``build_ext`` command. In general this is done inside the ``finalize_options`` + method with the help of the ``set_undefined_options`` command:: + + def finalize_options(self): + self.set_undefined_options("build_py", ("build_lib", "build_lib")) + ... + """ + + def initialize_options(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def finalize_options(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def run(self): + """(Required by the original :class:`setuptools.Command` interface)""" + + def get_source_files(self) -> List[str]: + """ + Return a list of all files that are used by the command to create the expected + outputs. + For example, if your build command transpiles Java files into Python, you should + list here all the Java files. + The primary purpose of this function is to help populating the ``sdist`` + with all the files necessary to build the distribution. + All files should be strings relative to the project root directory. + """ + + def get_outputs(self) -> List[str]: + """ + Return a list of files intended for distribution as they would have been + produced by the build. + These files should be strings in the form of + ``"{build_lib}/destination/file/path"``. + + .. note:: + The return value of ``get_output()`` should include all files used as keys + in ``get_output_mapping()`` plus files that are generated during the build + and don't correspond to any source file already present in the project. + """ + + def get_output_mapping(self) -> Dict[str, str]: + """ + Return a mapping between destination files as they would be produced by the + build (dict keys) into the respective existing (source) files (dict values). + Existing (source) files should be represented as strings relative to the project + root directory. + Destination files should be strings in the form of + ``"{build_lib}/destination/file/path"``. + """ diff --git a/setuptools/command/build_ext.py b/setuptools/command/build_ext.py index c59eff8bbf..7ad5a87adc 100644 --- a/setuptools/command/build_ext.py +++ b/setuptools/command/build_ext.py @@ -2,14 +2,16 @@ import sys import itertools from importlib.machinery import EXTENSION_SUFFIXES +from importlib.util import cache_from_source as _compiled_file_name +from typing import Dict, Iterator, List, Tuple + from distutils.command.build_ext import build_ext as _du_build_ext -from distutils.file_util import copy_file from distutils.ccompiler import new_compiler from distutils.sysconfig import customize_compiler, get_config_var -from distutils.errors import DistutilsError from distutils import log -from setuptools.extension import Library +from setuptools.errors import BaseError +from setuptools.extension import Extension, Library try: # Attempt to use Cython for building extensions, if available @@ -73,6 +75,9 @@ def get_abi3_suffix(): class build_ext(_build_ext): + editable_mode: bool = False + inplace: bool = False + def run(self): """Build extensions in build directory, then copy if --inplace""" old_inplace, self.inplace = self.inplace, 0 @@ -81,27 +86,61 @@ def run(self): if old_inplace: self.copy_extensions_to_source() + def _get_inplace_equivalent(self, build_py, ext: Extension) -> Tuple[str, str]: + fullname = self.get_ext_fullname(ext.name) + filename = self.get_ext_filename(fullname) + modpath = fullname.split('.') + package = '.'.join(modpath[:-1]) + package_dir = build_py.get_package_dir(package) + inplace_file = os.path.join(package_dir, os.path.basename(filename)) + regular_file = os.path.join(self.build_lib, filename) + return (inplace_file, regular_file) + def copy_extensions_to_source(self): build_py = self.get_finalized_command('build_py') for ext in self.extensions: - fullname = self.get_ext_fullname(ext.name) - filename = self.get_ext_filename(fullname) - modpath = fullname.split('.') - package = '.'.join(modpath[:-1]) - package_dir = build_py.get_package_dir(package) - dest_filename = os.path.join(package_dir, - os.path.basename(filename)) - src_filename = os.path.join(self.build_lib, filename) + inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext) # Always copy, even if source is older than destination, to ensure # that the right extensions for the current Python/platform are # used. - copy_file( - src_filename, dest_filename, verbose=self.verbose, - dry_run=self.dry_run - ) + self.copy_file(regular_file, inplace_file, level=self.verbose) + if ext._needs_stub: - self.write_stub(package_dir or os.curdir, ext, True) + inplace_stub = self._get_equivalent_stub(ext, inplace_file) + self._write_stub_file(inplace_stub, ext, compile=True) + # Always compile stub and remove the original (leave the cache behind) + # (this behaviour was observed in previous iterations of the code) + + def _get_equivalent_stub(self, ext: Extension, output_file: str) -> str: + dir_ = os.path.dirname(output_file) + _, _, name = ext.name.rpartition(".") + return f"{os.path.join(dir_, name)}.py" + + def _get_output_mapping(self) -> Iterator[Tuple[str, str]]: + if not self.inplace: + return + + build_py = self.get_finalized_command('build_py') + opt = self.get_finalized_command('install_lib').optimize or "" + + for ext in self.extensions: + inplace_file, regular_file = self._get_inplace_equivalent(build_py, ext) + yield (regular_file, inplace_file) + + if ext._needs_stub: + # This version of `build_ext` always builds artifacts in another dir, + # when "inplace=True" is given it just copies them back. + # This is done in the `copy_extensions_to_source` function, which + # always compile stub files via `_compile_and_remove_stub`. + # At the end of the process, a `.pyc` stub file is created without the + # corresponding `.py`. + + inplace_stub = self._get_equivalent_stub(ext, inplace_file) + regular_stub = self._get_equivalent_stub(ext, regular_file) + inplace_cache = _compiled_file_name(inplace_stub, optimization=opt) + output_cache = _compiled_file_name(regular_stub, optimization=opt) + yield (output_cache, inplace_cache) def get_ext_filename(self, fullname): so_ext = os.getenv('SETUPTOOLS_EXT_SUFFIX') @@ -131,6 +170,7 @@ def initialize_options(self): self.shlib_compiler = None self.shlibs = [] self.ext_map = {} + self.editable_mode = False def finalize_options(self): _build_ext.finalize_options(self) @@ -161,6 +201,9 @@ def finalize_options(self): if ltd and use_stubs and os.curdir not in ext.runtime_library_dirs: ext.runtime_library_dirs.append(os.curdir) + if self.editable_mode: + self.inplace = True + def setup_shlib_compiler(self): compiler = self.shlib_compiler = new_compiler( compiler=self.compiler, dry_run=self.dry_run, force=self.force @@ -201,8 +244,8 @@ def build_extension(self, ext): self.compiler = self.shlib_compiler _build_ext.build_extension(self, ext) if ext._needs_stub: - cmd = self.get_finalized_command('build_py').build_lib - self.write_stub(cmd, ext) + build_lib = self.get_finalized_command('build_py').build_lib + self.write_stub(build_lib, ext) finally: self.compiler = _compiler @@ -215,8 +258,15 @@ def links_to_dynamic(self, ext): pkg = '.'.join(ext._full_name.split('.')[:-1] + ['']) return any(pkg + libname in libnames for libname in ext.libraries) - def get_outputs(self): - return _build_ext.get_outputs(self) + self.__get_stubs_outputs() + def get_outputs(self) -> List[str]: + if self.inplace: + return list(self.get_output_mapping().keys()) + return sorted(_build_ext.get_outputs(self) + self.__get_stubs_outputs()) + + def get_output_mapping(self) -> Dict[str, str]: + """See :class:`setuptools.commands.build.SubCommand`""" + mapping = self._get_output_mapping() + return dict(sorted(mapping, key=lambda x: x[0])) def __get_stubs_outputs(self): # assemble the base name for each extension that needs a stub @@ -236,12 +286,13 @@ def __get_output_extensions(self): yield '.pyo' def write_stub(self, output_dir, ext, compile=False): - log.info("writing stub loader for %s to %s", ext._full_name, - output_dir) - stub_file = (os.path.join(output_dir, *ext._full_name.split('.')) + - '.py') + stub_file = os.path.join(output_dir, *ext._full_name.split('.')) + '.py' + self._write_stub_file(stub_file, ext, compile) + + def _write_stub_file(self, stub_file: str, ext: Extension, compile=False): + log.info("writing stub loader for %s to %s", ext._full_name, stub_file) if compile and os.path.exists(stub_file): - raise DistutilsError(stub_file + " already exists! Please delete.") + raise BaseError(stub_file + " already exists! Please delete.") if not self.dry_run: f = open(stub_file, 'w') f.write( @@ -274,16 +325,19 @@ def write_stub(self, output_dir, ext, compile=False): ) f.close() if compile: - from distutils.util import byte_compile + self._compile_and_remove_stub(stub_file) + + def _compile_and_remove_stub(self, stub_file: str): + from distutils.util import byte_compile - byte_compile([stub_file], optimize=0, + byte_compile([stub_file], optimize=0, + force=True, dry_run=self.dry_run) + optimize = self.get_finalized_command('install_lib').optimize + if optimize > 0: + byte_compile([stub_file], optimize=optimize, force=True, dry_run=self.dry_run) - optimize = self.get_finalized_command('install_lib').optimize - if optimize > 0: - byte_compile([stub_file], optimize=optimize, - force=True, dry_run=self.dry_run) - if os.path.exists(stub_file) and not self.dry_run: - os.unlink(stub_file) + if os.path.exists(stub_file) and not self.dry_run: + os.unlink(stub_file) if use_stubs or os.name == 'nt': diff --git a/setuptools/command/build_py.py b/setuptools/command/build_py.py index 2fced3d6d5..923a32329f 100644 --- a/setuptools/command/build_py.py +++ b/setuptools/command/build_py.py @@ -11,6 +11,8 @@ import stat import warnings from pathlib import Path +from typing import Dict, Iterator, List, Optional, Tuple + from setuptools._deprecation_warning import SetuptoolsDeprecationWarning from setuptools.extern.more_itertools import unique_everseen @@ -28,6 +30,8 @@ class build_py(orig.build_py): Also, this version of the 'build_py' command allows you to specify both 'py_modules' and 'packages' in the same setup operation. """ + editable_mode: bool = False + existing_egg_info_dir: Optional[str] = None #: Private API, internal use only. def finalize_options(self): orig.build_py.finalize_options(self) @@ -37,9 +41,19 @@ def finalize_options(self): del self.__dict__['data_files'] self.__updated_files = [] + def copy_file(self, infile, outfile, preserve_mode=1, preserve_times=1, + link=None, level=1): + # Overwrite base class to allow using links + if link: + infile = str(Path(infile).resolve()) + outfile = str(Path(outfile).resolve()) + return super().copy_file(infile, outfile, preserve_mode, preserve_times, + link, level) + def run(self): """Build modules, packages, and copy data files to build directory""" - if not self.py_modules and not self.packages: + # if self.editable_mode or not (self.py_modules and self.packages): + if not (self.py_modules or self.packages) or self.editable_mode: return if self.py_modules: @@ -112,16 +126,41 @@ def find_data_files(self, package, src_dir): ) return self.exclude_data_files(package, src_dir, files) - def build_package_data(self): - """Copy data files into build directory""" + def get_outputs(self, include_bytecode=1) -> List[str]: + """See :class:`setuptools.commands.build.SubCommand`""" + if self.editable_mode: + return list(self.get_output_mapping().keys()) + return super().get_outputs(include_bytecode) + + def get_output_mapping(self) -> Dict[str, str]: + """See :class:`setuptools.commands.build.SubCommand`""" + mapping = itertools.chain( + self._get_package_data_output_mapping(), + self._get_module_mapping(), + ) + return dict(sorted(mapping, key=lambda x: x[0])) + + def _get_module_mapping(self) -> Iterator[Tuple[str, str]]: + """Iterate over all modules producing (dest, src) pairs.""" + for (package, module, module_file) in self.find_all_modules(): + package = package.split('.') + filename = self.get_module_outfile(self.build_lib, package, module) + yield (filename, module_file) + + def _get_package_data_output_mapping(self) -> Iterator[Tuple[str, str]]: + """Iterate over package data producing (dest, src) pairs.""" for package, src_dir, build_dir, filenames in self.data_files: for filename in filenames: target = os.path.join(build_dir, filename) - self.mkpath(os.path.dirname(target)) srcfile = os.path.join(src_dir, filename) - outf, copied = self.copy_file(srcfile, target) - make_writable(target) - srcfile = os.path.abspath(srcfile) + yield (target, srcfile) + + def build_package_data(self): + """Copy data files into build directory""" + for target, srcfile in self._get_package_data_output_mapping(): + self.mkpath(os.path.dirname(target)) + _outf, _copied = self.copy_file(srcfile, target) + make_writable(target) def analyze_manifest(self): self.manifest_files = mf = {} @@ -132,10 +171,19 @@ def analyze_manifest(self): # Locate package source directory src_dirs[assert_relative(self.get_package_dir(package))] = package - self.run_command('egg_info') + if ( + getattr(self, 'existing_egg_info_dir', None) + and Path(self.existing_egg_info_dir, "SOURCES.txt").exists() + ): + manifest = Path(self.existing_egg_info_dir, "SOURCES.txt") + files = manifest.read_text(encoding="utf-8").splitlines() + else: + self.run_command('egg_info') + ei_cmd = self.get_finalized_command('egg_info') + files = ei_cmd.filelist.files + check = _IncludePackageDataAbuse() - ei_cmd = self.get_finalized_command('egg_info') - for path in ei_cmd.filelist.files: + for path in files: d, f = os.path.split(assert_relative(path)) prev = None oldf = f @@ -189,6 +237,8 @@ def check_package(self, package, package_dir): def initialize_options(self): self.packages_checked = {} orig.build_py.initialize_options(self) + self.editable_mode = False + self.existing_egg_info_dir = None def get_package_dir(self, package): res = orig.build_py.get_package_dir(self, package) diff --git a/setuptools/command/dist_info.py b/setuptools/command/dist_info.py index ca540ad119..0685c94596 100644 --- a/setuptools/command/dist_info.py +++ b/setuptools/command/dist_info.py @@ -5,12 +5,17 @@ import os import re +import shutil +import sys import warnings +from contextlib import contextmanager from inspect import cleandoc +from pathlib import Path from distutils.core import Command from distutils import log from setuptools.extern import packaging +from setuptools._deprecation_warning import SetuptoolsDeprecationWarning class dist_info(Command): @@ -19,28 +24,85 @@ class dist_info(Command): user_options = [ ('egg-base=', 'e', "directory containing .egg-info directories" - " (default: top of the source tree)"), + " (default: top of the source tree)" + " DEPRECATED: use --output-dir."), + ('output-dir=', 'o', "directory inside of which the .dist-info will be" + "created (default: top of the source tree)"), + ('tag-date', 'd', "Add date stamp (e.g. 20050528) to version number"), + ('tag-build=', 'b', "Specify explicit tag to add to version number"), + ('no-date', 'D', "Don't include date stamp [default]"), + ('keep-egg-info', None, "*TRANSITIONAL* will be removed in the future"), ] + boolean_options = ['tag-date', 'keep-egg-info'] + negative_opt = {'no-date': 'tag-date'} + def initialize_options(self): self.egg_base = None + self.output_dir = None + self.name = None + self.dist_info_dir = None + self.tag_date = None + self.tag_build = None + self.keep_egg_info = False def finalize_options(self): - pass + if self.egg_base: + msg = "--egg-base is deprecated for dist_info command. Use --output-dir." + warnings.warn(msg, SetuptoolsDeprecationWarning) + self.output_dir = self.egg_base or self.output_dir + + dist = self.distribution + project_dir = dist.src_root or os.curdir + self.output_dir = Path(self.output_dir or project_dir) + + egg_info = self.reinitialize_command("egg_info") + egg_info.egg_base = str(self.output_dir) + + if self.tag_date: + egg_info.tag_date = self.tag_date + else: + self.tag_date = egg_info.tag_date + + if self.tag_build: + egg_info.tag_build = self.tag_build + else: + self.tag_build = egg_info.tag_build - def run(self): - egg_info = self.get_finalized_command('egg_info') - egg_info.egg_base = self.egg_base egg_info.finalize_options() - egg_info.run() - name = _safe(self.distribution.get_name()) - version = _version(self.distribution.get_version()) - base = self.egg_base or os.curdir - dist_info_dir = os.path.join(base, f"{name}-{version}.dist-info") - log.info("creating '{}'".format(os.path.abspath(dist_info_dir))) + self.egg_info = egg_info + + name = _safe(dist.get_name()) + version = _version(dist.get_version()) + self.name = f"{name}-{version}" + self.dist_info_dir = os.path.join(self.output_dir, f"{self.name}.dist-info") + + @contextmanager + def _maybe_bkp_dir(self, dir_path: str, requires_bkp: bool): + if requires_bkp: + bkp_name = f"{dir_path}.__bkp__" + _rm(bkp_name, ignore_errors=True) + _copy(dir_path, bkp_name, dirs_exist_ok=True, symlinks=True) + try: + yield + finally: + _rm(dir_path, ignore_errors=True) + shutil.move(bkp_name, dir_path) + else: + yield + def run(self): + self.output_dir.mkdir(parents=True, exist_ok=True) + self.egg_info.run() + egg_info_dir = self.egg_info.egg_info + assert os.path.isdir(egg_info_dir), ".egg-info dir should have been created" + + log.info("creating '{}'".format(os.path.abspath(self.dist_info_dir))) bdist_wheel = self.get_finalized_command('bdist_wheel') - bdist_wheel.egg2dist(egg_info.egg_info, dist_info_dir) + + # TODO: if bdist_wheel if merged into setuptools, just add "keep_egg_info" there + with self._maybe_bkp_dir(egg_info_dir, self.keep_egg_info): + bdist_wheel.egg2dist(egg_info_dir, self.dist_info_dir) def _safe(component: str) -> str: @@ -67,3 +129,14 @@ def _version(version: str) -> str: """ warnings.warn(cleandoc(msg)) return _safe(v).strip("_") + + +def _rm(dir_name, **opts): + if os.path.isdir(dir_name): + shutil.rmtree(dir_name, **opts) + + +def _copy(src, dst, **opts): + if sys.version_info < (3, 8): + opts.pop("dirs_exist_ok", None) + shutil.copytree(src, dst, **opts) diff --git a/setuptools/command/editable_wheel.py b/setuptools/command/editable_wheel.py new file mode 100644 index 0000000000..a44d24beb8 --- /dev/null +++ b/setuptools/command/editable_wheel.py @@ -0,0 +1,776 @@ +""" +Create a wheel that, when installed, will make the source package 'editable' +(add it to the interpreter's path, including metadata) per PEP 660. Replaces +'setup.py develop'. + +.. note:: + One of the mechanisms briefly mentioned in PEP 660 to implement editable installs is + to create a separated directory inside ``build`` and use a .pth file to point to that + directory. In the context of this file such directory is referred as + *auxiliary build directory* or ``auxiliary_dir``. +""" + +import logging +import os +import re +import shutil +import sys +import traceback +import warnings +from contextlib import suppress +from enum import Enum +from inspect import cleandoc +from itertools import chain +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Tuple, + TypeVar, + Union, +) + +from setuptools import Command, SetuptoolsDeprecationWarning, errors, namespaces +from setuptools.discovery import find_package_path +from setuptools.dist import Distribution + +if TYPE_CHECKING: + from wheel.wheelfile import WheelFile # noqa + +if sys.version_info >= (3, 8): + from typing import Protocol +elif TYPE_CHECKING: + from typing_extensions import Protocol +else: + from abc import ABC as Protocol + +_Path = Union[str, Path] +_P = TypeVar("_P", bound=_Path) +_logger = logging.getLogger(__name__) + + +class _EditableMode(Enum): + """ + Possible editable installation modes: + `lenient` (new files automatically added to the package - DEFAULT); + `strict` (requires a new installation when files are added/removed); or + `compat` (attempts to emulate `python setup.py develop` - DEPRECATED). + """ + + STRICT = "strict" + LENIENT = "lenient" + COMPAT = "compat" # TODO: Remove `compat` after Dec/2022. + + @classmethod + def convert(cls, mode: Optional[str]) -> "_EditableMode": + if not mode: + return _EditableMode.LENIENT # default + + _mode = mode.upper() + if _mode not in _EditableMode.__members__: + raise errors.OptionError(f"Invalid editable mode: {mode!r}. Try: 'strict'.") + + if _mode == "COMPAT": + msg = """ + The 'compat' editable mode is transitional and will be removed + in future versions of `setuptools`. + Please adapt your code accordingly to use either the 'strict' or the + 'lenient' modes. + + For more information, please check: + https://setuptools.pypa.io/en/latest/userguide/development_mode.html + """ + warnings.warn(msg, SetuptoolsDeprecationWarning) + + return _EditableMode[_mode] + + +_STRICT_WARNING = """ +New or renamed files may not be automatically picked up without a new installation. +""" + +_LENIENT_WARNING = """ +Options like `package-data`, `include/exclude-package-data` or +`packages.find.exclude/include` may have no effect. +""" + + +class editable_wheel(Command): + """Build 'editable' wheel for development. + (This command is reserved for internal use of setuptools). + """ + + description = "create a PEP 660 'editable' wheel" + + user_options = [ + ("dist-dir=", "d", "directory to put final built distributions in"), + ("dist-info-dir=", "I", "path to a pre-build .dist-info directory"), + ("mode=", None, cleandoc(_EditableMode.__doc__ or "")), + ] + + def initialize_options(self): + self.dist_dir = None + self.dist_info_dir = None + self.project_dir = None + self.mode = None + + def finalize_options(self): + dist = self.distribution + self.project_dir = dist.src_root or os.curdir + self.package_dir = dist.package_dir or {} + self.dist_dir = Path(self.dist_dir or os.path.join(self.project_dir, "dist")) + + def run(self): + try: + self.dist_dir.mkdir(exist_ok=True) + self._ensure_dist_info() + + # Add missing dist_info files + bdist_wheel = self.reinitialize_command("bdist_wheel") + bdist_wheel.write_wheelfile(self.dist_info_dir) + + self._create_wheel_file(bdist_wheel) + except Exception as ex: + traceback.print_exc() + msg = """ + Support for editable installs via PEP 660 was recently introduced + in `setuptools`. If you are seeing this error, please report to: + + https://github.com/pypa/setuptools/issues + + Meanwhile you can try the legacy behavior by setting an + environment variable and trying to install again: + + SETUPTOOLS_ENABLE_FEATURES="legacy-editable" + """ + raise errors.InternalError(cleandoc(msg)) from ex + + def _ensure_dist_info(self): + if self.dist_info_dir is None: + dist_info = self.reinitialize_command("dist_info") + dist_info.output_dir = self.dist_dir + dist_info.finalize_options() + dist_info.run() + self.dist_info_dir = dist_info.dist_info_dir + else: + assert str(self.dist_info_dir).endswith(".dist-info") + assert Path(self.dist_info_dir, "METADATA").exists() + + def _install_namespaces(self, installation_dir, pth_prefix): + # XXX: Only required to support the deprecated namespace practice + dist = self.distribution + if not dist.namespace_packages: + return + + src_root = Path(self.project_dir, self.pakcage_dir.get("", ".")).resolve() + installer = _NamespaceInstaller(dist, installation_dir, pth_prefix, src_root) + installer.install_namespaces() + + def _find_egg_info_dir(self) -> Optional[str]: + parent_dir = Path(self.dist_info_dir).parent if self.dist_info_dir else Path() + candidates = map(str, parent_dir.glob("*.egg-info")) + return next(candidates, None) + + def _configure_build( + self, name: str, unpacked_wheel: _Path, build_lib: _Path, tmp_dir: _Path + ): + """Configure commands to behave in the following ways: + + - Build commands can write to ``build_lib`` if they really want to... + (but this folder is expected to be ignored and modules are expected to live + in the project directory...) + - Binary extensions should be built in-place (editable_mode = True) + - Data/header/script files are not part of the "editable" specification + so they are written directly to the unpacked_wheel directory. + """ + # Non-editable files (data, headers, scripts) are written directly to the + # unpacked_wheel + + dist = self.distribution + wheel = str(unpacked_wheel) + build_lib = str(build_lib) + data = str(Path(unpacked_wheel, f"{name}.data", "data")) + headers = str(Path(unpacked_wheel, f"{name}.data", "headers")) + scripts = str(Path(unpacked_wheel, f"{name}.data", "scripts")) + + # egg-info may be generated again to create a manifest (used for package data) + egg_info = dist.reinitialize_command("egg_info", reinit_subcommands=True) + egg_info.egg_base = str(tmp_dir) + egg_info.ignore_egg_info_in_manifest = True + + build = dist.reinitialize_command("build", reinit_subcommands=True) + install = dist.reinitialize_command("install", reinit_subcommands=True) + + build.build_platlib = build.build_purelib = build.build_lib = build_lib + install.install_purelib = install.install_platlib = install.install_lib = wheel + install.install_scripts = build.build_scripts = scripts + install.install_headers = headers + install.install_data = data + + install_scripts = dist.get_command_obj("install_scripts") + install_scripts.no_ep = True + + build.build_temp = str(tmp_dir) + + build_py = dist.get_command_obj("build_py") + build_py.compile = False + build_py.existing_egg_info_dir = self._find_egg_info_dir() + + self._set_editable_mode() + + build.ensure_finalized() + install.ensure_finalized() + + def _set_editable_mode(self): + """Set the ``editable_mode`` flag in the build sub-commands""" + dist = self.distribution + build = dist.get_command_obj("build") + for cmd_name in build.get_sub_commands(): + cmd = dist.get_command_obj(cmd_name) + if hasattr(cmd, "editable_mode"): + cmd.editable_mode = True + + def _collect_build_outputs(self) -> Tuple[List[str], Dict[str, str]]: + files: List[str] = [] + mapping: Dict[str, str] = {} + build = self.get_finalized_command("build") + + for cmd_name in build.get_sub_commands(): + cmd = self.get_finalized_command(cmd_name) + if hasattr(cmd, "get_outputs"): + files.extend(cmd.get_outputs() or []) + if hasattr(cmd, "get_output_mapping"): + mapping.update(cmd.get_output_mapping() or {}) + + return files, mapping + + def _run_build_commands( + self, dist_name: str, unpacked_wheel: _Path, build_lib: _Path, tmp_dir: _Path + ) -> Tuple[List[str], Dict[str, str]]: + self._configure_build(dist_name, unpacked_wheel, build_lib, tmp_dir) + self.run_command("build") + files, mapping = self._collect_build_outputs() + self._run_install("headers") + self._run_install("scripts") + self._run_install("data") + return files, mapping + + def _create_wheel_file(self, bdist_wheel): + from wheel.wheelfile import WheelFile + + dist_info = self.get_finalized_command("dist_info") + dist_name = dist_info.name + tag = "-".join(bdist_wheel.get_tag()) + build_tag = "0.editable" # According to PEP 427 needs to start with digit + archive_name = f"{dist_name}-{build_tag}-{tag}.whl" + wheel_path = Path(self.dist_dir, archive_name) + if wheel_path.exists(): + wheel_path.unlink() + + unpacked_wheel = TemporaryDirectory(suffix=archive_name) + build_lib = TemporaryDirectory(suffix=".build-lib") + build_tmp = TemporaryDirectory(suffix=".build-temp") + + with unpacked_wheel as unpacked, build_lib as lib, build_tmp as tmp: + unpacked_dist_info = Path(unpacked, Path(self.dist_info_dir).name) + shutil.copytree(self.dist_info_dir, unpacked_dist_info) + self._install_namespaces(unpacked, dist_info.name) + files, mapping = self._run_build_commands(dist_name, unpacked, lib, tmp) + strategy = self._select_strategy(dist_name, tag, lib) + with strategy, WheelFile(wheel_path, "w") as wheel_obj: + strategy(wheel_obj, files, mapping) + wheel_obj.write_files(unpacked) + + return wheel_path + + def _run_install(self, category: str): + has_category = getattr(self.distribution, f"has_{category}", None) + if has_category and has_category(): + _logger.info(f"Installing {category} as non editable") + self.run_command(f"install_{category}") + + def _select_strategy( + self, + name: str, + tag: str, + build_lib: _Path, + ) -> "EditableStrategy": + """Decides which strategy to use to implement an editable installation.""" + build_name = f"__editable__.{name}-{tag}" + project_dir = Path(self.project_dir) + mode = _EditableMode.convert(self.mode) + + if mode is _EditableMode.STRICT: + auxiliary_dir = _empty_dir(Path(self.project_dir, "build", build_name)) + return _LinkTree(self.distribution, name, auxiliary_dir, build_lib) + + packages = _find_packages(self.distribution) + has_simple_layout = _simple_layout(packages, self.package_dir, project_dir) + is_compat_mode = mode is _EditableMode.COMPAT + if set(self.package_dir) == {""} and has_simple_layout or is_compat_mode: + # src-layout(ish) is relatively safe for a simple pth file + src_dir = self.package_dir.get("", ".") + return _StaticPth(self.distribution, name, [Path(project_dir, src_dir)]) + + # Use a MetaPathFinder to avoid adding accidental top-level packages/modules + return _TopLevelFinder(self.distribution, name) + + +class EditableStrategy(Protocol): + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + ... + + def __enter__(self): + ... + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + + +class _StaticPth: + def __init__(self, dist: Distribution, name: str, path_entries: List[Path]): + self.dist = dist + self.name = name + self.path_entries = path_entries + + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + entries = "\n".join((str(p.resolve()) for p in self.path_entries)) + contents = bytes(f"{entries}\n", "utf-8") + wheel.writestr(f"__editable__.{self.name}.pth", contents) + + def __enter__(self): + msg = f""" + Editable install will be performed using .pth file to extend `sys.path` with: + {self.path_entries!r} + """ + _logger.warning(msg + _LENIENT_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + + +class _LinkTree(_StaticPth): + """ + Creates a ``.pth`` file that points to a link tree in the ``auxiliary_dir``. + + This strategy will only link files (not dirs), so it can be implemented in + any OS, even if that means using hardlinks instead of symlinks. + + By collocating ``auxiliary_dir`` and the original source code, limitations + with hardlinks should be avoided. + """ + def __init__( + self, dist: Distribution, + name: str, + auxiliary_dir: _Path, + build_lib: _Path, + ): + self.auxiliary_dir = Path(auxiliary_dir) + self.build_lib = Path(build_lib).resolve() + self._file = dist.get_command_obj("build_py").copy_file + super().__init__(dist, name, [self.auxiliary_dir]) + + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + self._create_links(files, mapping) + super().__call__(wheel, files, mapping) + + def _normalize_output(self, file: str) -> Optional[str]: + # Files relative to build_lib will be normalized to None + with suppress(ValueError): + path = Path(file).resolve().relative_to(self.build_lib) + return str(path).replace(os.sep, '/') + return None + + def _create_file(self, relative_output: str, src_file: str, link=None): + dest = self.auxiliary_dir / relative_output + if not dest.parent.is_dir(): + dest.parent.mkdir(parents=True) + self._file(src_file, dest, link=link) + + def _create_links(self, outputs, output_mapping): + self.auxiliary_dir.mkdir(parents=True, exist_ok=True) + link_type = "sym" if _can_symlink_files(self.auxiliary_dir) else "hard" + mappings = { + self._normalize_output(k): v + for k, v in output_mapping.items() + } + mappings.pop(None, None) # remove files that are not relative to build_lib + + for output in outputs: + relative = self._normalize_output(output) + if relative and relative not in mappings: + self._create_file(relative, output) + + for relative, src in mappings.items(): + self._create_file(relative, src, link=link_type) + + def __enter__(self): + msg = "Strict editable install will be performed using a link tree.\n" + _logger.warning(msg + _STRICT_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + msg = f"""\n + Strict editable installation performed using the auxiliary directory: + {self.auxiliary_dir} + + Please be careful to not remove this directory, otherwise you might not be able + to import/use your package. + """ + warnings.warn(msg, InformationOnly) + + +class _TopLevelFinder: + def __init__(self, dist: Distribution, name: str): + self.dist = dist + self.name = name + + def __call__(self, wheel: "WheelFile", files: List[str], mapping: Dict[str, str]): + src_root = self.dist.src_root or os.curdir + top_level = chain(_find_packages(self.dist), _find_top_level_modules(self.dist)) + package_dir = self.dist.package_dir or {} + roots = _find_package_roots(top_level, package_dir, src_root) + + namespaces_: Dict[str, List[str]] = dict(chain( + _find_namespaces(self.dist.packages, roots), + ((ns, []) for ns in _find_virtual_namespaces(roots)), + )) + + name = f"__editable__.{self.name}.finder" + finder = _make_identifier(name) + content = bytes(_finder_template(name, roots, namespaces_), "utf-8") + wheel.writestr(f"{finder}.py", content) + + content = bytes(f"import {finder}; {finder}.install()", "utf-8") + wheel.writestr(f"__editable__.{self.name}.pth", content) + + def __enter__(self): + msg = "Editable install will be performed using a meta path finder.\n" + _logger.warning(msg + _LENIENT_WARNING) + return self + + def __exit__(self, _exc_type, _exc_value, _traceback): + ... + + +def _can_symlink_files(base_dir: Path) -> bool: + with TemporaryDirectory(dir=str(base_dir.resolve())) as tmp: + path1, path2 = Path(tmp, "file1.txt"), Path(tmp, "file2.txt") + path1.write_text("file1", encoding="utf-8") + with suppress(AttributeError, NotImplementedError, OSError): + os.symlink(path1, path2) + if path2.is_symlink() and path2.read_text(encoding="utf-8") == "file1": + return True + + try: + os.link(path1, path2) # Ensure hard links can be created + except Exception as ex: + msg = ( + "File system does not seem to support either symlinks or hard links. " + "Strict editable installs require one of them to be supported." + ) + raise LinksNotSupported(msg) from ex + return False + + +def _simple_layout( + packages: Iterable[str], package_dir: Dict[str, str], project_dir: Path +) -> bool: + """Return ``True`` if: + - all packages are contained by the same parent directory, **and** + - all packages become importable if the parent directory is added to ``sys.path``. + + >>> _simple_layout(['a'], {"": "src"}, "/tmp/myproj") + True + >>> _simple_layout(['a', 'a.b'], {"": "src"}, "/tmp/myproj") + True + >>> _simple_layout(['a', 'a.b'], {}, "/tmp/myproj") + True + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"": "src"}, "/tmp/myproj") + True + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "a", "b": "b"}, ".") + True + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "_a", "b": "_b"}, ".") + False + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a": "_a"}, "/tmp/myproj") + False + >>> _simple_layout(['a', 'a.a1', 'a.a1.a2', 'b'], {"a.a1.a2": "_a2"}, ".") + False + >>> _simple_layout(['a', 'a.b'], {"": "src", "a.b": "_ab"}, "/tmp/myproj") + False + """ + layout = { + pkg: find_package_path(pkg, package_dir, project_dir) + for pkg in packages + } + if not layout: + return False + parent = os.path.commonpath([_parent_path(k, v) for k, v in layout.items()]) + return all( + _normalize_path(Path(parent, *key.split('.'))) == _normalize_path(value) + for key, value in layout.items() + ) + + +def _parent_path(pkg, pkg_path): + """Infer the parent path containing a package, that if added to ``sys.path`` would + allow importing that package. + When ``pkg`` is directly mapped into a directory with a different name, return its + own path. + >>> _parent_path("a", "src/a") + 'src' + >>> _parent_path("b", "src/c") + 'src/c' + """ + parent = pkg_path[:-len(pkg)] if pkg_path.endswith(pkg) else pkg_path + return parent.rstrip("/" + os.sep) + + +def _find_packages(dist: Distribution) -> Iterator[str]: + yield from iter(dist.packages or []) + + py_modules = dist.py_modules or [] + nested_modules = [mod for mod in py_modules if "." in mod] + if dist.ext_package: + yield dist.ext_package + else: + ext_modules = dist.ext_modules or [] + nested_modules += [x.name for x in ext_modules if "." in x.name] + + for module in nested_modules: + package, _, _ = module.rpartition(".") + yield package + + +def _find_top_level_modules(dist: Distribution) -> Iterator[str]: + py_modules = dist.py_modules or [] + yield from (mod for mod in py_modules if "." not in mod) + + if not dist.ext_package: + ext_modules = dist.ext_modules or [] + yield from (x.name for x in ext_modules if "." not in x.name) + + +def _find_package_roots( + packages: Iterable[str], + package_dir: Mapping[str, str], + src_root: _Path, +) -> Dict[str, str]: + pkg_roots: Dict[str, str] = { + pkg: _absolute_root(find_package_path(pkg, package_dir, src_root)) + for pkg in sorted(packages) + } + + return _remove_nested(pkg_roots) + + +def _absolute_root(path: _Path) -> str: + """Works for packages and top-level modules""" + path_ = Path(path) + parent = path_.parent + + if path_.exists(): + return str(path_.resolve()) + else: + return str(parent.resolve() / path_.name) + + +def _find_virtual_namespaces(pkg_roots: Dict[str, str]) -> Iterator[str]: + """By carefully designing ``package_dir``, it is possible to implement the logical + structure of PEP 420 in a package without the corresponding directories. + This function will try to find this kind of namespaces. + """ + for pkg in pkg_roots: + if "." not in pkg: + continue + parts = pkg.split(".") + for i in range(len(parts) - 1, 0, -1): + partial_name = ".".join(parts[:i]) + path = Path(find_package_path(partial_name, pkg_roots, "")) + if not path.exists(): + yield partial_name + + +def _find_namespaces( + packages: List[str], pkg_roots: Dict[str, str] +) -> Iterator[Tuple[str, List[str]]]: + for pkg in packages: + path = find_package_path(pkg, pkg_roots, "") + if Path(path).exists() and not Path(path, "__init__.py").exists(): + yield (pkg, [path]) + + +def _remove_nested(pkg_roots: Dict[str, str]) -> Dict[str, str]: + output = dict(pkg_roots.copy()) + + for pkg, path in reversed(list(pkg_roots.items())): + if any( + pkg != other and _is_nested(pkg, path, other, other_path) + for other, other_path in pkg_roots.items() + ): + output.pop(pkg) + + return output + + +def _is_nested(pkg: str, pkg_path: str, parent: str, parent_path: str) -> bool: + """ + Return ``True`` if ``pkg`` is nested inside ``parent`` both logically and in the + file system. + >>> _is_nested("a.b", "path/a/b", "a", "path/a") + True + >>> _is_nested("a.b", "path/a/b", "a", "otherpath/a") + False + >>> _is_nested("a.b", "path/a/b", "c", "path/c") + False + """ + norm_pkg_path = _normalize_path(pkg_path) + rest = pkg.replace(parent, "").strip(".").split(".") + return ( + pkg.startswith(parent) + and norm_pkg_path == _normalize_path(Path(parent_path, *rest)) + ) + + +def _normalize_path(filename: _Path) -> str: + """Normalize a file/dir name for comparison purposes""" + # See pkg_resources.normalize_path + file = os.path.abspath(filename) if sys.platform == 'cygwin' else filename + return os.path.normcase(os.path.realpath(os.path.normpath(file))) + + +def _empty_dir(dir_: _P) -> _P: + """Create a directory ensured to be empty. Existing files may be removed.""" + shutil.rmtree(dir_, ignore_errors=True) + os.makedirs(dir_) + return dir_ + + +def _make_identifier(name: str) -> str: + """Make a string safe to be used as Python identifier. + >>> _make_identifier("12abc") + '_12abc' + >>> _make_identifier("__editable__.myns.pkg-78.9.3_local") + '__editable___myns_pkg_78_9_3_local' + """ + safe = re.sub(r'\W|^(?=\d)', '_', name) + assert safe.isidentifier() + return safe + + +class _NamespaceInstaller(namespaces.Installer): + def __init__(self, distribution, installation_dir, editable_name, src_root): + self.distribution = distribution + self.src_root = src_root + self.installation_dir = installation_dir + self.editable_name = editable_name + self.outputs = [] + + def _get_target(self): + """Installation target.""" + return os.path.join(self.installation_dir, self.editable_name) + + def _get_root(self): + """Where the modules/packages should be loaded from.""" + return repr(str(self.src_root)) + + +_FINDER_TEMPLATE = """\ +import sys +from importlib.machinery import ModuleSpec +from importlib.machinery import all_suffixes as module_suffixes +from importlib.util import spec_from_file_location +from itertools import chain +from pathlib import Path + +MAPPING = {mapping!r} +NAMESPACES = {namespaces!r} +PATH_PLACEHOLDER = {name!r} + ".__path_hook__" + + +class _EditableFinder: # MetaPathFinder + @classmethod + def find_spec(cls, fullname, path=None, target=None): + for pkg, pkg_path in reversed(list(MAPPING.items())): + if fullname.startswith(pkg): + rest = fullname.replace(pkg, "").strip(".").split(".") + return cls._find_spec(fullname, Path(pkg_path, *rest)) + + return None + + @classmethod + def _find_spec(cls, fullname, candidate_path): + init = candidate_path / "__init__.py" + candidates = (candidate_path.with_suffix(x) for x in module_suffixes()) + for candidate in chain([init], candidates): + if candidate.exists(): + return spec_from_file_location(fullname, candidate) + + +class _EditableNamespaceFinder: # PathEntryFinder + @classmethod + def _path_hook(cls, path): + if path == PATH_PLACEHOLDER: + return cls + raise ImportError + + @classmethod + def _paths(cls, fullname): + # Ensure __path__ is not empty for the spec to be considered a namespace. + return NAMESPACES[fullname] or MAPPING.get(fullname) or [PATH_PLACEHOLDER] + + @classmethod + def find_spec(cls, fullname, target=None): + if fullname in NAMESPACES: + spec = ModuleSpec(fullname, None, is_package=True) + spec.submodule_search_locations = cls._paths(fullname) + return spec + return None + + @classmethod + def find_module(cls, fullname): + return None + + +def install(): + if not any(finder == _EditableFinder for finder in sys.meta_path): + sys.meta_path.append(_EditableFinder) + + if not NAMESPACES: + return + + if not any(hook == _EditableNamespaceFinder._path_hook for hook in sys.path_hooks): + # PathEntryFinder is needed to create NamespaceSpec without private APIS + sys.path_hooks.append(_EditableNamespaceFinder._path_hook) + if PATH_PLACEHOLDER not in sys.path: + sys.path.append(PATH_PLACEHOLDER) # Used just to trigger the path hook +""" + + +def _finder_template( + name: str, mapping: Mapping[str, str], namespaces: Dict[str, List[str]] +) -> str: + """Create a string containing the code for the``MetaPathFinder`` and + ``PathEntryFinder``. + """ + mapping = dict(sorted(mapping.items(), key=lambda p: p[0])) + return _FINDER_TEMPLATE.format(name=name, mapping=mapping, namespaces=namespaces) + + +class InformationOnly(UserWarning): + """Currently there is no clear way of displaying messages to the users + that use the setuptools backend directly via ``pip``. + The only thing that might work is a warning, although it is not the + most appropriate tool for the job... + """ + + +class LinksNotSupported(errors.FileError): + """File system does not seem to support either symlinks or hard links.""" diff --git a/setuptools/command/egg_info.py b/setuptools/command/egg_info.py index 42a0178fce..25888ed864 100644 --- a/setuptools/command/egg_info.py +++ b/setuptools/command/egg_info.py @@ -182,6 +182,7 @@ def initialize_options(self): self.egg_info = None self.egg_version = None self.broken_egg_info = False + self.ignore_egg_info_in_manifest = False #################################### # allow the 'tag_svn_revision' to be detected and @@ -310,6 +311,7 @@ def find_sources(self): """Generate SOURCES.txt manifest file""" manifest_filename = os.path.join(self.egg_info, "SOURCES.txt") mm = manifest_maker(self.distribution) + mm.ignore_egg_info_dir = self.ignore_egg_info_in_manifest mm.manifest = manifest_filename mm.run() self.filelist = mm.filelist @@ -333,6 +335,10 @@ def check_broken_egg_info(self): class FileList(_FileList): # Implementations of the various MANIFEST.in commands + def __init__(self, warn=None, debug_print=None, ignore_egg_info_dir=False): + super().__init__(warn, debug_print) + self.ignore_egg_info_dir = ignore_egg_info_dir + def process_template_line(self, line): # Parse the line: split it up, make sure the right number of words # is there, and return the relevant words. 'action' is always @@ -522,6 +528,10 @@ def _safe_path(self, path): return False try: + # ignore egg-info paths + is_egg_info = ".egg-info" in u_path or b".egg-info" in utf8_path + if self.ignore_egg_info_dir and is_egg_info: + return False # accept is either way checks out if os.path.exists(u_path) or os.path.exists(utf8_path): return True @@ -538,12 +548,13 @@ def initialize_options(self): self.prune = 1 self.manifest_only = 1 self.force_manifest = 1 + self.ignore_egg_info_dir = False def finalize_options(self): pass def run(self): - self.filelist = FileList() + self.filelist = FileList(ignore_egg_info_dir=self.ignore_egg_info_dir) if not os.path.exists(self.manifest): self.write_manifest() # it must exist so it'll get in the list self.add_defaults() diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py index 0ffeacf319..4a8cde7e16 100644 --- a/setuptools/command/sdist.py +++ b/setuptools/command/sdist.py @@ -4,10 +4,12 @@ import sys import io import contextlib +from itertools import chain from .py36compat import sdist_add_defaults from .._importlib import metadata +from .build import _ORIGINAL_SUBCOMMANDS _default_revctrl = list @@ -100,6 +102,10 @@ class NoValue: if orig_val is not NoValue: setattr(os, 'link', orig_val) + def add_defaults(self): + super().add_defaults() + self._add_defaults_build_sub_commands() + def _add_defaults_optional(self): super()._add_defaults_optional() if os.path.isfile('pyproject.toml'): @@ -112,6 +118,14 @@ def _add_defaults_python(self): self.filelist.extend(build_py.get_source_files()) self._add_data_files(self._safe_data_files(build_py)) + def _add_defaults_build_sub_commands(self): + build = self.get_finalized_command("build") + missing_cmds = set(build.get_sub_commands()) - _ORIGINAL_SUBCOMMANDS + # ^-- the original built-in sub-commands are already handled by default. + cmds = (self.get_finalized_command(c) for c in missing_cmds) + files = (c.get_source_files() for c in cmds if hasattr(c, "get_source_files")) + self.filelist.extend(chain.from_iterable(files)) + def _safe_data_files(self, build_py): """ Since the ``sdist`` class is also used to compute the MANIFEST diff --git a/setuptools/config/expand.py b/setuptools/config/expand.py index ed7564047a..384504d879 100644 --- a/setuptools/config/expand.py +++ b/setuptools/config/expand.py @@ -45,6 +45,8 @@ from distutils.errors import DistutilsOptionError +from .._path import same_path as _same_path + if TYPE_CHECKING: from setuptools.dist import Distribution # noqa from setuptools.discovery import ConfigDiscovery # noqa @@ -328,25 +330,6 @@ def find_packages( return packages -def _same_path(p1: _Path, p2: _Path) -> bool: - """Differs from os.path.samefile because it does not require paths to exist. - Purely string based (no comparison between i-nodes). - >>> _same_path("a/b", "./a/b") - True - >>> _same_path("a/b", "a/./b") - True - >>> _same_path("a/b", "././a/b") - True - >>> _same_path("a/b", "./a/b/c/..") - True - >>> _same_path("a/b", "../a/b/c") - False - >>> _same_path("a", "a/b") - False - """ - return os.path.normpath(p1) == os.path.normpath(p2) - - def _nest_path(parent: _Path, path: _Path) -> str: path = parent if path in {".", ""} else os.path.join(parent, path) return os.path.normpath(path) diff --git a/setuptools/discovery.py b/setuptools/discovery.py index 95c3c7f83e..6a3d2c9d2e 100644 --- a/setuptools/discovery.py +++ b/setuptools/discovery.py @@ -42,8 +42,18 @@ from fnmatch import fnmatchcase from glob import glob from pathlib import Path -from typing import TYPE_CHECKING -from typing import Callable, Dict, Iterator, Iterable, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Callable, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Tuple, + Union +) import _distutils_hack.override # noqa: F401 @@ -435,6 +445,7 @@ def _analyse_flat_modules(self) -> bool: def _ensure_no_accidental_inclusion(self, detected: List[str], kind: str): if len(detected) > 1: from inspect import cleandoc + from setuptools.errors import PackageDiscoveryError msg = f"""Multiple top-level {kind} discovered in a flat-layout: {detected}. @@ -527,7 +538,7 @@ def remove_stubs(packages: List[str]) -> List[str]: def find_parent_package( - packages: List[str], package_dir: Dict[str, str], root_dir: _Path + packages: List[str], package_dir: Mapping[str, str], root_dir: _Path ) -> Optional[str]: """Find the parent package that is not a namespace.""" packages = sorted(packages, key=len) @@ -550,7 +561,9 @@ def find_parent_package( return None -def find_package_path(name: str, package_dir: Dict[str, str], root_dir: _Path) -> str: +def find_package_path( + name: str, package_dir: Mapping[str, str], root_dir: _Path +) -> str: """Given a package name, return the path where it should be found on disk, considering the ``package_dir`` option. diff --git a/setuptools/extension.py b/setuptools/extension.py index f2bbd59df8..58c023f6b4 100644 --- a/setuptools/extension.py +++ b/setuptools/extension.py @@ -116,6 +116,9 @@ class Extension(_Extension): specifies that a build failure in the extension should not abort the build process, but simply not install the failing extension. + :keyword bool py_limited_api: + opt-in flag for the usage of :doc:`Python's limited API `. + :raises setuptools.errors.PlatformError: if 'runtime_library_dirs' is specified on Windows. (since v63) """ diff --git a/setuptools/tests/contexts.py b/setuptools/tests/contexts.py index 5894882435..7ddbc780fb 100644 --- a/setuptools/tests/contexts.py +++ b/setuptools/tests/contexts.py @@ -123,3 +123,26 @@ def session_locked_tmp_dir(request, tmp_path_factory, name): # ^-- prevent multiple workers to access the directory at once locked_dir.mkdir(exist_ok=True, parents=True) yield locked_dir + + +@contextlib.contextmanager +def save_paths(): + """Make sure ``sys.path``, ``sys.meta_path`` and ``sys.path_hooks`` are preserved""" + prev = sys.path[:], sys.meta_path[:], sys.path_hooks[:] + + try: + yield + finally: + sys.path, sys.meta_path, sys.path_hooks = prev + + +@contextlib.contextmanager +def save_sys_modules(): + """Make sure initial ``sys.modules`` is preserved""" + prev_modules = sys.modules + + try: + sys.modules = sys.modules.copy() + yield + finally: + sys.modules = prev_modules diff --git a/setuptools/tests/namespaces.py b/setuptools/tests/namespaces.py index 245cf8ea38..34e916f50a 100644 --- a/setuptools/tests/namespaces.py +++ b/setuptools/tests/namespaces.py @@ -28,6 +28,29 @@ def build_namespace_package(tmpdir, name): return src_dir +def build_pep420_namespace_package(tmpdir, name): + src_dir = tmpdir / name + src_dir.mkdir() + pyproject = src_dir / "pyproject.toml" + namespace, sep, rest = name.rpartition(".") + script = f"""\ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + + [project] + name = "{name}" + version = "3.14159" + """ + pyproject.write_text(textwrap.dedent(script), encoding='utf-8') + ns_pkg_dir = src_dir / namespace.replace(".", "/") + ns_pkg_dir.mkdir(parents=True) + pkg_mod = ns_pkg_dir / (rest + ".py") + some_functionality = f"name = {rest!r}" + pkg_mod.write_text(some_functionality, encoding='utf-8') + return src_dir + + def make_site_dir(target): """ Add a sitecustomize.py module in target to cause diff --git a/setuptools/tests/test_build_ext.py b/setuptools/tests/test_build_ext.py index 3177a2cdd6..07ebcaf82b 100644 --- a/setuptools/tests/test_build_ext.py +++ b/setuptools/tests/test_build_ext.py @@ -2,6 +2,7 @@ import sys import distutils.command.build_ext as orig from distutils.sysconfig import get_config_var +from importlib.util import cache_from_source as _compiled_file_name from jaraco import path @@ -83,6 +84,97 @@ def test_ext_suffix_override(self): finally: del os.environ['SETUPTOOLS_EXT_SUFFIX'] + def dist_with_example(self): + files = { + "src": {"mypkg": {"subpkg": {"ext2.c": ""}}}, + "c-extensions": {"ext1": {"main.c": ""}}, + } + + ext1 = Extension("mypkg.ext1", ["c-extensions/ext1/main.c"]) + ext2 = Extension("mypkg.subpkg.ext2", ["src/mypkg/subpkg/ext2.c"]) + ext3 = Extension("ext3", ["c-extension/ext3.c"]) + + path.build(files) + dist = Distribution({ + "script_name": "%test%", + "ext_modules": [ext1, ext2, ext3], + "package_dir": {"": "src"}, + }) + return dist + + def test_get_outputs(self, tmpdir_cwd, monkeypatch): + monkeypatch.setenv('SETUPTOOLS_EXT_SUFFIX', '.mp3') # make test OS-independent + monkeypatch.setattr('setuptools.command.build_ext.use_stubs', False) + dist = self.dist_with_example() + + # Regular build: get_outputs not empty, but get_output_mappings is empty + build_ext = dist.get_command_obj("build_ext") + build_ext.editable_mode = False + build_ext.ensure_finalized() + build_lib = build_ext.build_lib.replace(os.sep, "/") + outputs = [x.replace(os.sep, "/") for x in build_ext.get_outputs()] + assert outputs == [ + f"{build_lib}/ext3.mp3", + f"{build_lib}/mypkg/ext1.mp3", + f"{build_lib}/mypkg/subpkg/ext2.mp3", + ] + assert build_ext.get_output_mapping() == {} + + # Editable build: get_output_mappings should contain everything in get_outputs + dist.reinitialize_command("build_ext") + build_ext.editable_mode = True + build_ext.ensure_finalized() + mapping = { + k.replace(os.sep, "/"): v.replace(os.sep, "/") + for k, v in build_ext.get_output_mapping().items() + } + assert mapping == { + f"{build_lib}/ext3.mp3": "src/ext3.mp3", + f"{build_lib}/mypkg/ext1.mp3": "src/mypkg/ext1.mp3", + f"{build_lib}/mypkg/subpkg/ext2.mp3": "src/mypkg/subpkg/ext2.mp3", + } + + def test_get_output_mapping_with_stub(self, tmpdir_cwd, monkeypatch): + monkeypatch.setenv('SETUPTOOLS_EXT_SUFFIX', '.mp3') # make test OS-independent + monkeypatch.setattr('setuptools.command.build_ext.use_stubs', True) + dist = self.dist_with_example() + + # Editable build should create compiled stubs (.pyc files only, no .py) + build_ext = dist.get_command_obj("build_ext") + build_ext.editable_mode = True + build_ext.ensure_finalized() + for ext in build_ext.extensions: + monkeypatch.setattr(ext, "_needs_stub", True) + + build_lib = build_ext.build_lib.replace(os.sep, "/") + mapping = { + k.replace(os.sep, "/"): v.replace(os.sep, "/") + for k, v in build_ext.get_output_mapping().items() + } + + def C(file): + """Make it possible to do comparisons and tests in a OS-independent way""" + return _compiled_file_name(file).replace(os.sep, "/") + + assert mapping == { + C(f"{build_lib}/ext3.py"): C("src/ext3.py"), + f"{build_lib}/ext3.mp3": "src/ext3.mp3", + C(f"{build_lib}/mypkg/ext1.py"): C("src/mypkg/ext1.py"), + f"{build_lib}/mypkg/ext1.mp3": "src/mypkg/ext1.mp3", + C(f"{build_lib}/mypkg/subpkg/ext2.py"): C("src/mypkg/subpkg/ext2.py"), + f"{build_lib}/mypkg/subpkg/ext2.mp3": "src/mypkg/subpkg/ext2.mp3", + } + + # Ensure only the compiled stubs are present not the raw .py stub + assert f"{build_lib}/mypkg/ext1.py" not in mapping + assert f"{build_lib}/mypkg/subpkg/ext2.py" not in mapping + + # Visualize what the cached stub files look like + example_stub = C(f"{build_lib}/mypkg/ext1.py") + assert example_stub in mapping + assert example_stub.startswith(f"{build_lib}/mypkg/__pycache__/ext1") + assert example_stub.endswith(".pyc") + def test_build_ext_config_handling(tmpdir_cwd): files = { diff --git a/setuptools/tests/test_build_meta.py b/setuptools/tests/test_build_meta.py index 36940e768f..e70c71bdd0 100644 --- a/setuptools/tests/test_build_meta.py +++ b/setuptools/tests/test_build_meta.py @@ -8,6 +8,7 @@ from concurrent import futures import re from zipfile import ZipFile +from pathlib import Path import pytest from jaraco import path @@ -611,6 +612,71 @@ def test_build_sdist_relative_path_import(self, tmpdir_cwd): with pytest.raises(ImportError, match="^No module named 'hello'$"): build_backend.build_sdist("temp") + _simple_pyproject_example = { + "pyproject.toml": DALS(""" + [project] + name = "proj" + version = "42" + """), + "src": { + "proj": {"__init__.py": ""} + } + } + + def _assert_link_tree(self, parent_dir): + """All files in the directory should be either links or hard links""" + files = list(Path(parent_dir).glob("**/*")) + assert files # Should not be empty + for file in files: + assert file.is_symlink() or os.stat(file).st_nlink > 0 + + @pytest.mark.filterwarnings("ignore::setuptools.SetuptoolsDeprecationWarning") + # Since the backend is running via a process pool, in some operating systems + # we may have problems to make assertions based on warnings/stdout/stderr... + # So the best is to ignore them for the time being. + def test_editable_with_global_option_still_works(self, tmpdir_cwd): + """The usage of --global-option is now discouraged in favour of --build-option. + This is required to make more sense of the provided scape hatch and align with + previous pip behaviour. See pypa/setuptools#1928. + """ + path.build({**self._simple_pyproject_example, '_meta': {}}) + build_backend = self.get_build_backend() + assert not Path("build").exists() + + cfg = {"--global-option": ["--mode", "strict"]} + build_backend.prepare_metadata_for_build_editable("_meta", cfg) + build_backend.build_editable("temp", cfg, "_meta") + + self._assert_link_tree(next(Path("build").glob("__editable__.*"))) + + def test_editable_without_config_settings(self, tmpdir_cwd): + """ + Sanity check to ensure tests with --mode=strict are different from the ones + without --mode. + + --mode=strict should create a local directory with a package tree. + The directory should not get created otherwise. + """ + path.build(self._simple_pyproject_example) + build_backend = self.get_build_backend() + assert not Path("build").exists() + build_backend.build_editable("temp") + assert not Path("build").exists() + + @pytest.mark.parametrize( + "config_settings", [ + {"--build-option": ["--mode", "strict"]}, + {"editable-mode": "strict"}, + ] + ) + def test_editable_with_config_settings(self, tmpdir_cwd, config_settings): + path.build({**self._simple_pyproject_example, '_meta': {}}) + assert not Path("build").exists() + build_backend = self.get_build_backend() + build_backend.prepare_metadata_for_build_editable("_meta", config_settings) + build_backend.build_editable("temp", config_settings, "_meta") + self._assert_link_tree(next(Path("build").glob("__editable__.*"))) + @pytest.mark.parametrize('setup_literal, requirements', [ ("'foo'", ['foo']), ("['foo']", ['foo']), @@ -764,3 +830,27 @@ def test_sys_argv_passthrough(self, tmpdir_cwd): build_backend = self.get_build_backend() build_backend.build_sdist("temp") + + +def test_legacy_editable_install(venv, tmpdir, tmpdir_cwd): + pyproject = """ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + [project] + name = "myproj" + version = "42" + """ + path.build({"pyproject.toml": DALS(pyproject), "mymod.py": ""}) + + # First: sanity check + cmd = ["pip", "install", "--no-build-isolation", "-e", "."] + output = str(venv.run(cmd, cwd=tmpdir), "utf-8").lower() + assert "running setup.py develop for myproj" not in output + assert "created wheel for myproj" in output + + # Then: real test + env = {**os.environ, "SETUPTOOLS_ENABLE_FEATURES": "legacy-editable"} + cmd = ["pip", "install", "--no-build-isolation", "-e", "."] + output = str(venv.run(cmd, cwd=tmpdir, env=env), "utf-8").lower() + assert "running setup.py develop for myproj" in output diff --git a/setuptools/tests/test_build_py.py b/setuptools/tests/test_build_py.py index 13fa64de9e..77738f23fa 100644 --- a/setuptools/tests/test_build_py.py +++ b/setuptools/tests/test_build_py.py @@ -1,10 +1,11 @@ import os import stat import shutil +from pathlib import Path +from unittest.mock import Mock import pytest import jaraco.path -from path import Path from setuptools import SetuptoolsDeprecationWarning from setuptools.dist import Distribution @@ -109,67 +110,194 @@ def test_executable_data(tmpdir_cwd): "Script is not executable" -def test_excluded_subpackages(tmp_path): - files = { - "setup.cfg": DALS(""" - [metadata] - name = mypkg - version = 42 +EXAMPLE_WITH_MANIFEST = { + "setup.cfg": DALS(""" + [metadata] + name = mypkg + version = 42 - [options] - include_package_data = True - packages = find: + [options] + include_package_data = True + packages = find: - [options.packages.find] - exclude = *.tests* - """), + [options.packages.find] + exclude = *.tests* + """), + "mypkg": { + "__init__.py": "", + "resource_file.txt": "", + "tests": { + "__init__.py": "", + "test_mypkg.py": "", + "test_file.txt": "", + } + }, + "MANIFEST.in": DALS(""" + global-include *.py *.txt + global-exclude *.py[cod] + prune dist + prune build + prune *.egg-info + """) +} + + +def test_excluded_subpackages(tmpdir_cwd): + jaraco.path.build(EXAMPLE_WITH_MANIFEST) + dist = Distribution({"script_name": "%PEP 517%"}) + dist.parse_config_files() + + build_py = dist.get_command_obj("build_py") + msg = r"Python recognizes 'mypkg\.tests' as an importable package" + with pytest.warns(SetuptoolsDeprecationWarning, match=msg): + # TODO: To fix #3260 we need some transition period to deprecate the + # existing behavior of `include_package_data`. After the transition, we + # should remove the warning and fix the behaviour. + build_py.finalize_options() + build_py.run() + + build_dir = Path(dist.get_command_obj("build_py").build_lib) + assert (build_dir / "mypkg/__init__.py").exists() + assert (build_dir / "mypkg/resource_file.txt").exists() + + # Setuptools is configured to ignore `mypkg.tests`, therefore the following + # files/dirs should not be included in the distribution. + for f in [ + "mypkg/tests/__init__.py", + "mypkg/tests/test_mypkg.py", + "mypkg/tests/test_file.txt", + "mypkg/tests", + ]: + with pytest.raises(AssertionError): + # TODO: Enforce the following assertion once #3260 is fixed + # (remove context manager and the following xfail). + assert not (build_dir / f).exists() + + pytest.xfail("#3260") + + +@pytest.mark.filterwarnings("ignore::setuptools.SetuptoolsDeprecationWarning") +def test_existing_egg_info(tmpdir_cwd, monkeypatch): + """When provided with the ``existing_egg_info_dir`` attribute, build_py should not + attempt to run egg_info again. + """ + # == Pre-condition == + # Generate an egg-info dir + jaraco.path.build(EXAMPLE_WITH_MANIFEST) + dist = Distribution({"script_name": "%PEP 517%"}) + dist.parse_config_files() + assert dist.include_package_data + + egg_info = dist.get_command_obj("egg_info") + dist.run_command("egg_info") + egg_info_dir = next(Path(egg_info.egg_base).glob("*.egg-info")) + assert egg_info_dir.is_dir() + + # == Setup == + build_py = dist.get_command_obj("build_py") + build_py.finalize_options() + egg_info = dist.get_command_obj("egg_info") + egg_info_run = Mock(side_effect=egg_info.run) + monkeypatch.setattr(egg_info, "run", egg_info_run) + + # == Remove caches == + # egg_info is called when build_py looks for data_files, which gets cached. + # We need to ensure it is not cached yet, otherwise it may impact on the tests + build_py.__dict__.pop('data_files', None) + dist.reinitialize_command(egg_info) + + # == Sanity check == + # Ensure that if existing_egg_info is not given, build_py attempts to run egg_info + build_py.existing_egg_info_dir = None + build_py.run() + egg_info_run.assert_called() + + # == Remove caches == + egg_info_run.reset_mock() + build_py.__dict__.pop('data_files', None) + dist.reinitialize_command(egg_info) + + # == Actual test == + # Ensure that if existing_egg_info_dir is given, egg_info doesn't run + build_py.existing_egg_info_dir = egg_info_dir + build_py.run() + egg_info_run.assert_not_called() + assert build_py.data_files + + # Make sure the list of outputs is actually OK + outputs = map(lambda x: x.replace(os.sep, "/"), build_py.get_outputs()) + assert outputs + example = str(Path(build_py.build_lib, "mypkg/__init__.py")).replace(os.sep, "/") + assert example in outputs + + +EXAMPLE_ARBITRARY_MAPPING = { + "pyproject.toml": DALS(""" + [project] + name = "mypkg" + version = "42" + + [tool.setuptools] + packages = ["mypkg", "mypkg.sub1", "mypkg.sub2", "mypkg.sub2.nested"] + + [tool.setuptools.package-dir] + "" = "src" + "mypkg.sub2" = "src/mypkg/_sub2" + "mypkg.sub2.nested" = "other" + """), + "src": { "mypkg": { "__init__.py": "", "resource_file.txt": "", - "tests": { + "sub1": { "__init__.py": "", - "test_mypkg.py": "", - "test_file.txt": "", - } + "mod1.py": "", + }, + "_sub2": { + "mod2.py": "", + }, }, - "MANIFEST.in": DALS(""" - global-include *.py *.txt - global-exclude *.py[cod] - prune dist - prune build - prune *.egg-info - """) - } + }, + "other": { + "__init__.py": "", + "mod3.py": "", + }, + "MANIFEST.in": DALS(""" + global-include *.py *.txt + global-exclude *.py[cod] + """) +} + + +def test_get_outputs(tmpdir_cwd): + jaraco.path.build(EXAMPLE_ARBITRARY_MAPPING) + dist = Distribution({"script_name": "%test%"}) + dist.parse_config_files() - with Path(tmp_path): - jaraco.path.build(files) - dist = Distribution({"script_name": "%PEP 517%"}) - dist.parse_config_files() - - build_py = dist.get_command_obj("build_py") - msg = r"Python recognizes 'mypkg\.tests' as an importable package" - with pytest.warns(SetuptoolsDeprecationWarning, match=msg): - # TODO: To fix #3260 we need some transition period to deprecate the - # existing behavior of `include_package_data`. After the transition, we - # should remove the warning and fix the behaviour. - build_py.finalize_options() - build_py.run() - - build_dir = Path(dist.get_command_obj("build_py").build_lib) - assert (build_dir / "mypkg/__init__.py").exists() - assert (build_dir / "mypkg/resource_file.txt").exists() - - # Setuptools is configured to ignore `mypkg.tests`, therefore the following - # files/dirs should not be included in the distribution. - for f in [ - "mypkg/tests/__init__.py", - "mypkg/tests/test_mypkg.py", - "mypkg/tests/test_file.txt", - "mypkg/tests", - ]: - with pytest.raises(AssertionError): - # TODO: Enforce the following assertion once #3260 is fixed - # (remove context manager and the following xfail). - assert not (build_dir / f).exists() - - pytest.xfail("#3260") + build_py = dist.get_command_obj("build_py") + build_py.editable_mode = True + build_py.ensure_finalized() + build_lib = build_py.build_lib.replace(os.sep, "/") + outputs = {x.replace(os.sep, "/") for x in build_py.get_outputs()} + assert outputs == { + f"{build_lib}/mypkg/__init__.py", + f"{build_lib}/mypkg/resource_file.txt", + f"{build_lib}/mypkg/sub1/__init__.py", + f"{build_lib}/mypkg/sub1/mod1.py", + f"{build_lib}/mypkg/sub2/mod2.py", + f"{build_lib}/mypkg/sub2/nested/__init__.py", + f"{build_lib}/mypkg/sub2/nested/mod3.py", + } + mapping = { + k.replace(os.sep, "/"): v.replace(os.sep, "/") + for k, v in build_py.get_output_mapping().items() + } + assert mapping == { + f"{build_lib}/mypkg/__init__.py": "src/mypkg/__init__.py", + f"{build_lib}/mypkg/resource_file.txt": "src/mypkg/resource_file.txt", + f"{build_lib}/mypkg/sub1/__init__.py": "src/mypkg/sub1/__init__.py", + f"{build_lib}/mypkg/sub1/mod1.py": "src/mypkg/sub1/mod1.py", + f"{build_lib}/mypkg/sub2/mod2.py": "src/mypkg/_sub2/mod2.py", + f"{build_lib}/mypkg/sub2/nested/__init__.py": "other/__init__.py", + f"{build_lib}/mypkg/sub2/nested/mod3.py": "other/mod3.py", + } diff --git a/setuptools/tests/test_develop.py b/setuptools/tests/test_develop.py index c52072ac1e..0dd60342ba 100644 --- a/setuptools/tests/test_develop.py +++ b/setuptools/tests/test_develop.py @@ -5,12 +5,10 @@ import sys import subprocess import platform -import pathlib from setuptools.command import test import pytest -import pip_run.launch from setuptools.command.develop import develop from setuptools.dist import Distribution @@ -165,45 +163,3 @@ def test_namespace_package_importable(self, tmpdir): ] with test.test.paths_on_pythonpath([str(target)]): subprocess.check_call(pkg_resources_imp) - - @pytest.mark.xfail( - platform.python_implementation() == 'PyPy', - reason="Workaround fails on PyPy (why?)", - ) - def test_editable_prefix(self, tmp_path, sample_project): - """ - Editable install to a prefix should be discoverable. - """ - prefix = tmp_path / 'prefix' - - # figure out where pip will likely install the package - site_packages = prefix / next( - pathlib.Path(path).relative_to(sys.prefix) - for path in sys.path - if 'site-packages' in path and path.startswith(sys.prefix) - ) - site_packages.mkdir(parents=True) - - # install workaround - pip_run.launch.inject_sitecustomize(str(site_packages)) - - env = dict(os.environ, PYTHONPATH=str(site_packages)) - cmd = [ - sys.executable, - '-m', - 'pip', - 'install', - '--editable', - str(sample_project), - '--prefix', - str(prefix), - '--no-build-isolation', - ] - subprocess.check_call(cmd, env=env) - - # now run 'sample' with the prefix on the PYTHONPATH - bin = 'Scripts' if platform.system() == 'Windows' else 'bin' - exe = prefix / bin / 'sample' - if sys.version_info < (3, 8) and platform.system() == 'Windows': - exe = str(exe) - subprocess.check_call([exe], env=env) diff --git a/setuptools/tests/test_dist_info.py b/setuptools/tests/test_dist_info.py index 813ef51d32..350e6429a9 100644 --- a/setuptools/tests/test_dist_info.py +++ b/setuptools/tests/test_dist_info.py @@ -2,6 +2,7 @@ """ import pathlib import re +import shutil import subprocess import sys from functools import partial @@ -91,6 +92,42 @@ def test_invalid_version(self, tmp_path): dist_info = next(tmp_path.glob("*.dist-info")) assert dist_info.name.startswith("proj-42") + def test_tag_arguments(self, tmp_path): + config = """ + [metadata] + name=proj + version=42 + [egg_info] + tag_date=1 + tag_build=.post + """ + (tmp_path / "setup.cfg").write_text(config, encoding="utf-8") + + print(run_command("dist_info", "--no-date", cwd=tmp_path)) + dist_info = next(tmp_path.glob("*.dist-info")) + assert dist_info.name.startswith("proj-42") + shutil.rmtree(dist_info) + + print(run_command("dist_info", "--tag-build", ".a", cwd=tmp_path)) + dist_info = next(tmp_path.glob("*.dist-info")) + assert dist_info.name.startswith("proj-42a") + + @pytest.mark.parametrize("keep_egg_info", (False, True)) + def test_output_dir(self, tmp_path, keep_egg_info): + config = "[metadata]\nname=proj\nversion=42\n" + (tmp_path / "setup.cfg").write_text(config, encoding="utf-8") + out = (tmp_path / "__out") + out.mkdir() + opts = ["--keep-egg-info"] if keep_egg_info else [] + run_command("dist_info", "--output-dir", out, *opts, cwd=tmp_path) + assert len(list(out.glob("*.dist-info"))) == 1 + assert len(list(tmp_path.glob("*.dist-info"))) == 0 + expected_egg_info = 1 if keep_egg_info else 0 + assert len(list(out.glob("*.egg-info"))) == expected_egg_info + assert len(list(tmp_path.glob("*.egg-info"))) == 0 + assert len(list(out.glob("*.__bkp__"))) == 0 + assert len(list(tmp_path.glob("*.__bkp__"))) == 0 + class TestWheelCompatibility: """Make sure the .dist-info directory produced with the ``dist_info`` command @@ -154,5 +191,5 @@ def test_dist_info_is_the_same_as_in_wheel( def run_command(*cmd, **kwargs): opts = {"stderr": subprocess.STDOUT, "text": True, **kwargs} - cmd = [sys.executable, "-c", "__import__('setuptools').setup()", *cmd] + cmd = [sys.executable, "-c", "__import__('setuptools').setup()", *map(str, cmd)] return subprocess.check_output(cmd, **opts) diff --git a/setuptools/tests/test_editable_install.py b/setuptools/tests/test_editable_install.py index aac4f5eef0..57e31edabd 100644 --- a/setuptools/tests/test_editable_install.py +++ b/setuptools/tests/test_editable_install.py @@ -1,27 +1,45 @@ +import os +import stat +import sys import subprocess +import platform +from copy import deepcopy +from importlib import import_module +from pathlib import Path from textwrap import dedent +from unittest.mock import Mock +from uuid import uuid4 -import pytest import jaraco.envs -import path +import jaraco.path +import pip_run.launch +import pytest +from path import Path as _Path +from . import contexts, namespaces + +from setuptools._importlib import resources as importlib_resources +from setuptools.command.editable_wheel import ( + _LinkTree, + _find_virtual_namespaces, + _find_namespaces, + _find_package_roots, + _finder_template, +) +from setuptools.dist import Distribution -@pytest.fixture -def venv(tmp_path, setuptools_wheel): - env = jaraco.envs.VirtualEnv() - vars(env).update( - root=path.Path(tmp_path), # workaround for error on windows - name=".venv", - create_opts=["--no-setuptools"], - req=str(setuptools_wheel), - ) - return env.create() + +@pytest.fixture(params=["strict", "lenient"]) +def editable_opts(request): + if request.param == "strict": + return ["--config-settings", "editable-mode=strict"] + return [] EXAMPLE = { 'pyproject.toml': dedent("""\ [build-system] - requires = ["setuptools", "wheel"] + requires = ["setuptools"] build-backend = "setuptools.build_meta" [project] @@ -51,6 +69,8 @@ def venv(tmp_path, setuptools_wheel): "MANIFEST.in": dedent("""\ global-include *.py *.txt global-exclude *.py[cod] + prune dist + prune build """).strip(), "README.rst": "This is a ``README``", "LICENSE.txt": "---- placeholder MIT license ----", @@ -85,24 +105,23 @@ def venv(tmp_path, setuptools_wheel): SETUP_SCRIPT_STUB = "__import__('setuptools').setup()" -MISSING_SETUP_SCRIPT = pytest.param( - None, - marks=pytest.mark.xfail( - reason="Editable install is currently only supported with `setup.py`" - ) -) -@pytest.mark.parametrize("setup_script", [SETUP_SCRIPT_STUB, MISSING_SETUP_SCRIPT]) -def test_editable_with_pyproject(tmp_path, venv, setup_script): +@pytest.mark.parametrize( + "files", + [ + {**EXAMPLE, "setup.py": SETUP_SCRIPT_STUB}, # type: ignore + EXAMPLE, # No setup.py script + ] +) +def test_editable_with_pyproject(tmp_path, venv, files, editable_opts): project = tmp_path / "mypkg" - files = {**EXAMPLE, "setup.py": setup_script} project.mkdir() jaraco.path.build(files, prefix=project) cmd = [venv.exe(), "-m", "pip", "install", "--no-build-isolation", # required to force current version of setuptools - "-e", str(project)] + "-e", str(project), *editable_opts] print(str(subprocess.check_output(cmd), "utf-8")) cmd = [venv.exe(), "-m", "mypkg"] @@ -111,3 +130,590 @@ def test_editable_with_pyproject(tmp_path, venv, setup_script): (project / "src/mypkg/data.txt").write_text("foobar") (project / "src/mypkg/mod.py").write_text("x = 42") assert subprocess.check_output(cmd).strip() == b"3.14159.post0 foobar 42" + + +def test_editable_with_flat_layout(tmp_path, venv, editable_opts): + files = { + "mypkg": { + "pyproject.toml": dedent("""\ + [build-system] + requires = ["setuptools", "wheel"] + build-backend = "setuptools.build_meta" + + [project] + name = "mypkg" + version = "3.14159" + + [tool.setuptools] + packages = ["pkg"] + py-modules = ["mod"] + """), + "pkg": {"__init__.py": "a = 4"}, + "mod.py": "b = 2", + }, + } + jaraco.path.build(files, prefix=tmp_path) + project = tmp_path / "mypkg" + + cmd = [venv.exe(), "-m", "pip", "install", + "--no-build-isolation", # required to force current version of setuptools + "-e", str(project), *editable_opts] + print(str(subprocess.check_output(cmd), "utf-8")) + cmd = [venv.exe(), "-c", "import pkg, mod; print(pkg.a, mod.b)"] + assert subprocess.check_output(cmd).strip() == b"4 2" + + +class TestLegacyNamespaces: + """Ported from test_develop""" + + def test_namespace_package_importable(self, venv, tmp_path, editable_opts): + """ + Installing two packages sharing the same namespace, one installed + naturally using pip or `--single-version-externally-managed` + and the other installed in editable mode should leave the namespace + intact and both packages reachable by import. + """ + pkg_A = namespaces.build_namespace_package(tmp_path, 'myns.pkgA') + pkg_B = namespaces.build_namespace_package(tmp_path, 'myns.pkgB') + # use pip to install to the target directory + opts = editable_opts[:] + opts.append("--no-build-isolation") # force current version of setuptools + venv.run(["python", "-m", "pip", "install", str(pkg_A), *opts]) + venv.run(["python", "-m", "pip", "install", "-e", str(pkg_B), *opts]) + venv.run(["python", "-c", "import myns.pkgA; import myns.pkgB"]) + # additionally ensure that pkg_resources import works + venv.run(["python", "-c", "import pkg_resources"]) + + +class TestPep420Namespaces: + def test_namespace_package_importable(self, venv, tmp_path, editable_opts): + """ + Installing two packages sharing the same namespace, one installed + normally using pip and the other installed in editable mode + should allow importing both packages. + """ + pkg_A = namespaces.build_pep420_namespace_package(tmp_path, 'myns.n.pkgA') + pkg_B = namespaces.build_pep420_namespace_package(tmp_path, 'myns.n.pkgB') + # use pip to install to the target directory + opts = editable_opts[:] + opts.append("--no-build-isolation") # force current version of setuptools + venv.run(["python", "-m", "pip", "install", str(pkg_A), *opts]) + venv.run(["python", "-m", "pip", "install", "-e", str(pkg_B), *opts]) + venv.run(["python", "-c", "import myns.n.pkgA; import myns.n.pkgB"]) + + def test_namespace_created_via_package_dir(self, venv, tmp_path, editable_opts): + """Currently users can create a namespace by tweaking `package_dir`""" + files = { + "pkgA": { + "pyproject.toml": dedent("""\ + [build-system] + requires = ["setuptools", "wheel"] + build-backend = "setuptools.build_meta" + + [project] + name = "pkgA" + version = "3.14159" + + [tool.setuptools] + package-dir = {"myns.n.pkgA" = "src"} + """), + "src": {"__init__.py": "a = 1"}, + }, + } + jaraco.path.build(files, prefix=tmp_path) + pkg_A = tmp_path / "pkgA" + pkg_B = namespaces.build_pep420_namespace_package(tmp_path, 'myns.n.pkgB') + pkg_C = namespaces.build_pep420_namespace_package(tmp_path, 'myns.n.pkgC') + + # use pip to install to the target directory + opts = editable_opts[:] + opts.append("--no-build-isolation") # force current version of setuptools + venv.run(["python", "-m", "pip", "install", str(pkg_A), *opts]) + venv.run(["python", "-m", "pip", "install", "-e", str(pkg_B), *opts]) + venv.run(["python", "-m", "pip", "install", "-e", str(pkg_C), *opts]) + venv.run(["python", "-c", "from myns.n import pkgA, pkgB, pkgC"]) + + +# Moved here from test_develop: +@pytest.mark.xfail( + platform.python_implementation() == 'PyPy', + reason="Workaround fails on PyPy (why?)", +) +def test_editable_with_prefix(tmp_path, sample_project, editable_opts): + """ + Editable install to a prefix should be discoverable. + """ + prefix = tmp_path / 'prefix' + + # figure out where pip will likely install the package + site_packages = prefix / next( + Path(path).relative_to(sys.prefix) + for path in sys.path + if 'site-packages' in path and path.startswith(sys.prefix) + ) + site_packages.mkdir(parents=True) + + # install workaround + pip_run.launch.inject_sitecustomize(str(site_packages)) + + env = dict(os.environ, PYTHONPATH=str(site_packages)) + cmd = [ + sys.executable, + '-m', + 'pip', + 'install', + '--editable', + str(sample_project), + '--prefix', + str(prefix), + '--no-build-isolation', + *editable_opts, + ] + subprocess.check_call(cmd, env=env) + + # now run 'sample' with the prefix on the PYTHONPATH + bin = 'Scripts' if platform.system() == 'Windows' else 'bin' + exe = prefix / bin / 'sample' + if sys.version_info < (3, 8) and platform.system() == 'Windows': + exe = str(exe) + subprocess.check_call([exe], env=env) + + +class TestFinderTemplate: + """This test focus in getting a particular implementation detail right. + If at some point in time the implementation is changed for something different, + this test can be modified or even excluded. + """ + def install_finder(self, finder): + loc = {} + exec(finder, loc, loc) + loc["install"]() + + def test_packages(self, tmp_path): + files = { + "src1": { + "pkg1": { + "__init__.py": "", + "subpkg": {"mod1.py": "a = 42"}, + }, + }, + "src2": {"mod2.py": "a = 43"}, + } + jaraco.path.build(files, prefix=tmp_path) + + mapping = { + "pkg1": str(tmp_path / "src1/pkg1"), + "mod2": str(tmp_path / "src2/mod2") + } + template = _finder_template(str(uuid4()), mapping, {}) + + with contexts.save_paths(), contexts.save_sys_modules(): + for mod in ("pkg1", "pkg1.subpkg", "pkg1.subpkg.mod1", "mod2"): + sys.modules.pop(mod, None) + + self.install_finder(template) + mod1 = import_module("pkg1.subpkg.mod1") + mod2 = import_module("mod2") + subpkg = import_module("pkg1.subpkg") + + assert mod1.a == 42 + assert mod2.a == 43 + expected = str((tmp_path / "src1/pkg1/subpkg").resolve()) + assert_path(subpkg, expected) + + def test_namespace(self, tmp_path): + files = {"pkg": {"__init__.py": "a = 13", "text.txt": "abc"}} + jaraco.path.build(files, prefix=tmp_path) + + mapping = {"ns.othername": str(tmp_path / "pkg")} + namespaces = {"ns": []} + + template = _finder_template(str(uuid4()), mapping, namespaces) + with contexts.save_paths(), contexts.save_sys_modules(): + for mod in ("ns", "ns.othername"): + sys.modules.pop(mod, None) + + self.install_finder(template) + pkg = import_module("ns.othername") + text = importlib_resources.files(pkg) / "text.txt" + + expected = str((tmp_path / "pkg").resolve()) + assert_path(pkg, expected) + assert pkg.a == 13 + + # Make sure resources can also be found + assert text.read_text(encoding="utf-8") == "abc" + + def test_combine_namespaces(self, tmp_path): + files = { + "src1": {"ns": {"pkg1": {"__init__.py": "a = 13"}}}, + "src2": {"ns": {"mod2.py": "b = 37"}}, + } + jaraco.path.build(files, prefix=tmp_path) + + mapping = { + "ns.pkgA": str(tmp_path / "src1/ns/pkg1"), + "ns": str(tmp_path / "src2/ns"), + } + namespaces_ = {"ns": [str(tmp_path / "src1"), str(tmp_path / "src2")]} + template = _finder_template(str(uuid4()), mapping, namespaces_) + + with contexts.save_paths(), contexts.save_sys_modules(): + for mod in ("ns", "ns.pkgA", "ns.mod2"): + sys.modules.pop(mod, None) + + self.install_finder(template) + pkgA = import_module("ns.pkgA") + mod2 = import_module("ns.mod2") + + expected = str((tmp_path / "src1/ns/pkg1").resolve()) + assert_path(pkgA, expected) + assert pkgA.a == 13 + assert mod2.b == 37 + + def test_dynamic_path_computation(self, tmp_path): + # Follows the example in PEP 420 + files = { + "project1": {"parent": {"child": {"one.py": "x = 1"}}}, + "project2": {"parent": {"child": {"two.py": "x = 2"}}}, + "project3": {"parent": {"child": {"three.py": "x = 3"}}}, + } + jaraco.path.build(files, prefix=tmp_path) + mapping = {} + namespaces_ = {"parent": [str(tmp_path / "project1/parent")]} + template = _finder_template(str(uuid4()), mapping, namespaces_) + + mods = (f"parent.child.{name}" for name in ("one", "two", "three")) + with contexts.save_paths(), contexts.save_sys_modules(): + for mod in ("parent", "parent.child", "parent.child", *mods): + sys.modules.pop(mod, None) + + self.install_finder(template) + + one = import_module("parent.child.one") + assert one.x == 1 + + with pytest.raises(ImportError): + import_module("parent.child.two") + + sys.path.append(str(tmp_path / "project2")) + two = import_module("parent.child.two") + assert two.x == 2 + + with pytest.raises(ImportError): + import_module("parent.child.three") + + sys.path.append(str(tmp_path / "project3")) + three = import_module("parent.child.three") + assert three.x == 3 + + +def test_pkg_roots(tmp_path): + """This test focus in getting a particular implementation detail right. + If at some point in time the implementation is changed for something different, + this test can be modified or even excluded. + """ + files = { + "a": {"b": {"__init__.py": "ab = 1"}, "__init__.py": "a = 1"}, + "d": {"__init__.py": "d = 1", "e": {"__init__.py": "de = 1"}}, + "f": {"g": {"h": {"__init__.py": "fgh = 1"}}}, + "other": {"__init__.py": "abc = 1"}, + "another": {"__init__.py": "abcxyz = 1"}, + "yet_another": {"__init__.py": "mnopq = 1"}, + } + jaraco.path.build(files, prefix=tmp_path) + package_dir = { + "a.b.c": "other", + "a.b.c.x.y.z": "another", + "m.n.o.p.q": "yet_another" + } + packages = [ + "a", + "a.b", + "a.b.c", + "a.b.c.x.y", + "a.b.c.x.y.z", + "d", + "d.e", + "f", + "f.g", + "f.g.h", + "m.n.o.p.q", + ] + roots = _find_package_roots(packages, package_dir, tmp_path) + assert roots == { + "a": str(tmp_path / "a"), + "a.b.c": str(tmp_path / "other"), + "a.b.c.x.y.z": str(tmp_path / "another"), + "d": str(tmp_path / "d"), + "f": str(tmp_path / "f"), + "m.n.o.p.q": str(tmp_path / "yet_another"), + } + + ns = set(dict(_find_namespaces(packages, roots))) + assert ns == {"f", "f.g"} + + ns = set(_find_virtual_namespaces(roots)) + assert ns == {"a.b.c.x", "a.b.c.x.y", "m", "m.n", "m.n.o", "m.n.o.p"} + + +class TestOverallBehaviour: + PYPROJECT = """\ + [build-system] + requires = ["setuptools"] + build-backend = "setuptools.build_meta" + + [project] + name = "mypkg" + version = "3.14159" + """ + + FLAT_LAYOUT = { + "pyproject.toml": dedent(PYPROJECT), + "MANIFEST.in": EXAMPLE["MANIFEST.in"], + "otherfile.py": "", + "mypkg": { + "__init__.py": "", + "mod1.py": "var = 42", + "subpackage": { + "__init__.py": "", + "mod2.py": "var = 13", + "resource_file.txt": "resource 39", + }, + }, + } + + EXAMPLES = { + "flat-layout": FLAT_LAYOUT, + "src-layout": { + "pyproject.toml": dedent(PYPROJECT), + "MANIFEST.in": EXAMPLE["MANIFEST.in"], + "otherfile.py": "", + "src": {"mypkg": FLAT_LAYOUT["mypkg"]}, + }, + "custom-layout": { + "pyproject.toml": dedent(PYPROJECT) + dedent("""\ + [tool.setuptools] + packages = ["mypkg", "mypkg.subpackage"] + + [tool.setuptools.package-dir] + "mypkg.subpackage" = "other" + """), + "MANIFEST.in": EXAMPLE["MANIFEST.in"], + "otherfile.py": "", + "mypkg": { + "__init__.py": "", + "mod1.py": FLAT_LAYOUT["mypkg"]["mod1.py"], # type: ignore + }, + "other": FLAT_LAYOUT["mypkg"]["subpackage"], # type: ignore + }, + "namespace": { + "pyproject.toml": dedent(PYPROJECT), + "MANIFEST.in": EXAMPLE["MANIFEST.in"], + "otherfile.py": "", + "src": { + "mypkg": { + "mod1.py": FLAT_LAYOUT["mypkg"]["mod1.py"], # type: ignore + "subpackage": FLAT_LAYOUT["mypkg"]["subpackage"], # type: ignore + }, + }, + }, + } + + @pytest.mark.parametrize("layout", EXAMPLES.keys()) + def test_editable_install(self, tmp_path, venv, layout, editable_opts): + opts = editable_opts + project = install_project("mypkg", venv, tmp_path, self.EXAMPLES[layout], *opts) + + # Ensure stray files are not importable + cmd_import_error = """\ + try: + import otherfile + except ImportError as ex: + print(ex) + """ + out = venv.run(["python", "-c", dedent(cmd_import_error)]) + assert b"No module named 'otherfile'" in out + + # Ensure the modules are importable + cmd_get_vars = """\ + import mypkg, mypkg.mod1, mypkg.subpackage.mod2 + print(mypkg.mod1.var, mypkg.subpackage.mod2.var) + """ + out = venv.run(["python", "-c", dedent(cmd_get_vars)]) + assert b"42 13" in out + + # Ensure resources are reachable + cmd_get_resource = """\ + import mypkg.subpackage + from setuptools._importlib import resources as importlib_resources + text = importlib_resources.files(mypkg.subpackage) / "resource_file.txt" + print(text.read_text(encoding="utf-8")) + """ + out = venv.run(["python", "-c", dedent(cmd_get_resource)]) + assert b"resource 39" in out + + # Ensure files are editable + mod1 = next(project.glob("**/mod1.py")) + mod2 = next(project.glob("**/mod2.py")) + resource_file = next(project.glob("**/resource_file.txt")) + + mod1.write_text("var = 17", encoding="utf-8") + mod2.write_text("var = 781", encoding="utf-8") + resource_file.write_text("resource 374", encoding="utf-8") + + out = venv.run(["python", "-c", dedent(cmd_get_vars)]) + assert b"42 13" not in out + assert b"17 781" in out + + out = venv.run(["python", "-c", dedent(cmd_get_resource)]) + assert b"resource 39" not in out + assert b"resource 374" in out + + +class TestLinkTree: + FILES = deepcopy(TestOverallBehaviour.EXAMPLES["src-layout"]) + FILES["pyproject.toml"] += dedent("""\ + [tool.setuptools] + # Temporary workaround: both `include-package-data` and `package-data` configs + # can be removed after #3260 is fixed. + include-package-data = false + package-data = {"*" = ["*.txt"]} + + [tool.setuptools.packages.find] + where = ["src"] + exclude = ["*.subpackage*"] + """) + FILES["src"]["mypkg"]["resource.not_in_manifest"] = "abc" + + def test_generated_tree(self, tmp_path): + jaraco.path.build(self.FILES, prefix=tmp_path) + + with _Path(tmp_path): + name = "mypkg-3.14159" + dist = Distribution({"script_name": "%PEP 517%"}) + dist.parse_config_files() + + wheel = Mock() + aux = tmp_path / ".aux" + build = tmp_path / ".build" + aux.mkdir() + build.mkdir() + + build_py = dist.get_command_obj("build_py") + build_py.editable_mode = True + build_py.build_lib = str(build) + build_py.ensure_finalized() + outputs = build_py.get_outputs() + output_mapping = build_py.get_output_mapping() + + make_tree = _LinkTree(dist, name, aux, build) + make_tree(wheel, outputs, output_mapping) + + mod1 = next(aux.glob("**/mod1.py")) + expected = tmp_path / "src/mypkg/mod1.py" + assert_link_to(mod1, expected) + + assert next(aux.glob("**/subpackage"), None) is None + assert next(aux.glob("**/mod2.py"), None) is None + assert next(aux.glob("**/resource_file.txt"), None) is None + + assert next(aux.glob("**/resource.not_in_manifest"), None) is None + + def test_strict_install(self, tmp_path, venv): + opts = ["--config-settings", "editable-mode=strict"] + install_project("mypkg", venv, tmp_path, self.FILES, *opts) + + out = venv.run(["python", "-c", "import mypkg.mod1; print(mypkg.mod1.var)"]) + assert b"42" in out + + # Ensure packages excluded from distribution are not importable + cmd_import_error = """\ + try: + from mypkg import subpackage + except ImportError as ex: + print(ex) + """ + out = venv.run(["python", "-c", dedent(cmd_import_error)]) + assert b"cannot import name 'subpackage'" in out + + # Ensure resource files excluded from distribution are not reachable + cmd_get_resource = """\ + import mypkg + from setuptools._importlib import resources as importlib_resources + try: + text = importlib_resources.files(mypkg) / "resource.not_in_manifest" + print(text.read_text(encoding="utf-8")) + except FileNotFoundError as ex: + print(ex) + """ + out = venv.run(["python", "-c", dedent(cmd_get_resource)]) + assert b"No such file or directory" in out + assert b"resource.not_in_manifest" in out + + +@pytest.mark.filterwarnings("ignore:.*compat.*:setuptools.SetuptoolsDeprecationWarning") +def test_compat_install(tmp_path, venv): + # TODO: Remove `compat` after Dec/2022. + opts = ["--config-settings", "editable-mode=compat"] + files = TestOverallBehaviour.EXAMPLES["custom-layout"] + install_project("mypkg", venv, tmp_path, files, *opts) + + out = venv.run(["python", "-c", "import mypkg.mod1; print(mypkg.mod1.var)"]) + assert b"42" in out + + expected_path = comparable_path(str(tmp_path)) + + # Compatible behaviour will make spurious modules and excluded + # files importable directly from the original path + for cmd in ( + "import otherfile; print(otherfile)", + "import other; print(other)", + "import mypkg; print(mypkg)", + ): + out = comparable_path(str(venv.run(["python", "-c", cmd]), "utf-8")) + assert expected_path in out + + # Compatible behaviour will not consider custom mappings + cmd = """\ + try: + from mypkg import subpackage; + except ImportError as ex: + print(ex) + """ + out = str(venv.run(["python", "-c", dedent(cmd)]), "utf-8") + assert "cannot import name 'subpackage'" in out + + +def install_project(name, venv, tmp_path, files, *opts): + project = tmp_path / name + project.mkdir() + jaraco.path.build(files, prefix=project) + opts = [*opts, "--no-build-isolation"] # force current version of setuptools + venv.run(["python", "-m", "pip", "install", "-e", str(project), *opts]) + return project + + +# ---- Assertion Helpers ---- + + +def assert_path(pkg, expected): + # __path__ is not guaranteed to exist, so we have to account for that + if pkg.__path__: + path = next(iter(pkg.__path__), None) + if path: + assert str(Path(path).resolve()) == expected + + +def assert_link_to(file: Path, other: Path): + if file.is_symlink(): + assert str(file.resolve()) == str(other.resolve()) + else: + file_stat = file.stat() + other_stat = other.stat() + assert file_stat[stat.ST_INO] == other_stat[stat.ST_INO] + assert file_stat[stat.ST_DEV] == other_stat[stat.ST_DEV] + + +def comparable_path(str_with_path: str) -> str: + return str_with_path.lower().replace(os.sep, "/").replace("//", "/") diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py index 302cff7309..4b0d2e17c2 100644 --- a/setuptools/tests/test_sdist.py +++ b/setuptools/tests/test_sdist.py @@ -10,6 +10,7 @@ import pytest +from setuptools import Command from setuptools._importlib import metadata from setuptools import SetuptoolsDeprecationWarning from setuptools.command.sdist import sdist @@ -517,6 +518,46 @@ def test_pyproject_toml_excluded(self, tmpdir): manifest = cmd.filelist.files assert 'pyproject.toml' not in manifest + def test_build_subcommand_source_files(self, tmpdir): + touch(tmpdir / '.myfile~') + + # Sanity check: without custom commands file list should not be affected + dist = Distribution({**SETUP_ATTRS, "script_name": "setup.py"}) + cmd = sdist(dist) + cmd.ensure_finalized() + with quiet(): + cmd.run() + manifest = cmd.filelist.files + assert '.myfile~' not in manifest + + # Test: custom command should be able to augment file list + dist = Distribution({**SETUP_ATTRS, "script_name": "setup.py"}) + build = dist.get_command_obj("build") + build.sub_commands = [*build.sub_commands, ("build_custom", None)] + + class build_custom(Command): + def initialize_options(self): + ... + + def finalize_options(self): + ... + + def run(self): + ... + + def get_source_files(self): + return ['.myfile~'] + + dist.cmdclass.update(build_custom=build_custom) + + cmd = sdist(dist) + cmd.use_defaults = True + cmd.ensure_finalized() + with quiet(): + cmd.run() + manifest = cmd.filelist.files + assert '.myfile~' in manifest + def test_default_revctrl(): """