diff --git a/changelog.d/2887.change.1.rst b/changelog.d/2887.change.1.rst new file mode 100644 index 0000000000..11d7a7169c --- /dev/null +++ b/changelog.d/2887.change.1.rst @@ -0,0 +1,20 @@ +Added automatic discovery for ``py_modules`` and ``packages`` +-- by :user:`abravalheri`. + +Setuptools will try to find these values assuming that the package uses either +the *src-layout* (a ``src`` directory containing all the packages or modules), +the *flat-layout* (package directories directly under the project root), +or the *single-module* approach (isolated Python files, directly under +the project root). + +The automatic discovery will also respect layouts that are explicitly +configured using the ``package_dir`` option. + +For backward-compatibility, this behavior will be observed **only if both** +``py_modules`` **and** ``packages`` **are not set**. + +If setuptools detects modules or packages that are not supposed to be in the +distribution, please manually set ``py_modules`` and ``packages`` in your +``setup.cfg`` or ``setup.py`` file. +If you are using a *flat-layout*, you can also consider switching to +*src-layout*. diff --git a/changelog.d/2887.change.2.rst b/changelog.d/2887.change.2.rst new file mode 100644 index 0000000000..a6aa041a3b --- /dev/null +++ b/changelog.d/2887.change.2.rst @@ -0,0 +1,9 @@ +Added automatic configuration for the ``name`` metadata +-- by :user:`abravalheri`. + +Setuptools will adopt the name of the top-level package (or module in the case +of single-module distributions), **only when** ``name`` **is not explicitly +provided**. + +Please note that it is not possible to automatically derive a single name when +the distribution consists of multiple top-level packages or modules. diff --git a/changelog.d/2894.breaking.rst b/changelog.d/2894.breaking.rst new file mode 100644 index 0000000000..687ae511a6 --- /dev/null +++ b/changelog.d/2894.breaking.rst @@ -0,0 +1,10 @@ +If you purposefully want to create an *"empty distribution"*, please be aware +that some Python files (or general folders) might be automatically detected and +included. + +Projects that currently don't specify both ``packages`` and ``py_modules`` in their +configuration and have extra Python files and folders (not meant for distribution), +might see these files being included in the wheel archive. + +You can check details about the automatic discovery behaviour (and +how to configure a different one) in :doc:`/userguide/package_discovery`. diff --git a/docs/userguide/package_discovery.rst b/docs/userguide/package_discovery.rst index 61da2d662c..99e45bedd7 100644 --- a/docs/userguide/package_discovery.rst +++ b/docs/userguide/package_discovery.rst @@ -38,8 +38,142 @@ included manually in the following manner: packages=['mypkg1', 'mypkg2'] ) -This can get tiresome really quickly. To speed things up, we introduce two -functions provided by setuptools: +This can get tiresome really quickly. To speed things up, you can rely on +setuptools automatic discovery, or use the provided tools, as explained in +the following sections. + + +Automatic discovery +=================== + +By default setuptools will consider 2 popular project layouts, each one with +its own set of advantages and disadvantages [#layout1]_ [#layout2]_. + +src-layout: + The project should contain a ``src`` directory under the project root and + all modules and packages meant for distribution are placed inside this + directory:: + + project_root_directory + ├── pyproject.toml + ├── setup.cfg # or setup.py + ├── ... + └── src/ + └── mypkg/ + ├── __init__.py + ├── ... + └── mymodule.py + + This layout is very handy when you wish to use automatic discovery, + since you don't have to worry about other Python files or folders in your + project root being distributed by mistake. In some circumstances it can be + also less error-prone for testing or when using :pep:`420`-style packages. + On the other hand you cannot rely on the implicit ``PYTHONPATH=.`` to fire + up the Python REPL and play with your package (you will need an + `editable install`_ to be able to do that). + +flat-layout (also known as "adhoc"): + The package folder(s) are placed directly under the project root:: + + project_root_directory + ├── pyproject.toml + ├── setup.cfg # or setup.py + ├── ... + └── mypkg/ + ├── __init__.py + ├── ... + └── mymodule.py + + This layout is very practical for using the REPL, but in some situations + it can be can be more error-prone (e.g. during tests or if you have a bunch + of folders or Python files hanging around your project root) + +There is also a handy variation of the *flat-layout* for utilities/libraries +that can be implemented with a single Python file: + +single-module approach (or "few top-level modules"): + Standalone modules are placed directly under the project root, instead of + inside a package folder:: + + project_root_directory + ├── pyproject.toml + ├── setup.cfg # or setup.py + ├── ... + └── single_file_lib.py + +Setuptools will automatically scan your project directory looking for these +layouts and try to guess the correct values for the :ref:`packages ` and :doc:`py_modules ` configuration. + +To avoid confusion, file and folder names that are used by popular tools (or +that correspond to well-known conventions, such as distributing documentation +alongside the project code) are automatically filtered out in the case of +*flat-layouts*: + +.. autoattribute:: setuptools.discovery.FlatLayoutPackageFinder.DEFAULT_EXCLUDE + +.. autoattribute:: setuptools.discovery.FlatLayoutModuleFinder.DEFAULT_EXCLUDE + +Also note that you can customise your project layout by explicitly setting +``package_dir``: + +.. tab:: setup.cfg + + .. code-block:: ini + + [options] + # ... + package_dir = + = lib + # similar to "src-layout" but using the "lib" folder + # pkg.mod corresponds to lib/pkg/mod.py + # OR + package_dir = + pkg1 = lib1 + # pkg1.mod corresponds to lib1/mod.py + # pkg1.subpkg.mod corresponds to lib1/subpkg/mod.py + pkg2 = lib2 + # pkg2.mod corresponds to lib2/mod.py + pkg2.subpkg = lib3 + # pkg2.subpkg.mod corresponds to lib3/mod.py + +.. tab:: setup.py + + .. code-block:: python + + setup( + # ... + package_dir = {"": "lib"} + # similar to "src-layout" but using the "lib" folder + # pkg.mod corresponds to lib/pkg/mod.py + ) + + # OR + + setup( + # ... + package_dir = { + "pkg1": "lib1", # pkg1.mod corresponds to lib1/mod.py + # pkg1.subpkg.mod corresponds to lib1/subpkg/mod.py + "pkg2": "lib2", # pkg2.mod corresponds to lib2/mod.py + "pkg2.subpkg": "lib3" # pkg2.subpkg.mod corresponds to lib3/mod.py + # ... + ) + +.. important:: Automatic discovery will **only** be enabled if you don't + provide any configuration for both ``packages`` and ``py_modules``. + If at least one of them is explicitly set, automatic discovery will not take + place. + + +Custom discovery +================ + +If the automatic discovery does not work for you +(e.g., you want to *include* in the distribution top-level packages with +reserved names such as ``tasks``, ``example`` or ``docs``, or you want to +*exclude* nested packages that would be otherwise included), you can use +the provided tools for package discovery: .. tab:: setup.cfg @@ -61,7 +195,7 @@ functions provided by setuptools: Using ``find:`` or ``find_packages`` -==================================== +------------------------------------ Let's start with the first tool. ``find:`` (``find_packages``) takes a source directory and two lists of package name patterns to exclude and include, and then return a list of ``str`` representing the packages it could find. To use @@ -113,7 +247,7 @@ in ``src`` that starts with the name ``pkg`` and not ``additional``: .. _Namespace Packages: Using ``find_namespace:`` or ``find_namespace_packages`` -======================================================== +-------------------------------------------------------- ``setuptools`` provides the ``find_namespace:`` (``find_namespace_packages``) which behaves similarly to ``find:`` but works with namespace package. Before diving in, it is important to have a good understanding of what namespace @@ -249,3 +383,9 @@ file contains the following: __path__ = __import__('pkgutil').extend_path(__path__, __name__) The project layout remains the same and ``setup.cfg`` remains the same. + + +.. [#layout1] https://blog.ionelmc.ro/2014/05/25/python-packaging/#the-structure +.. [#layout2] https://blog.ionelmc.ro/2017/09/25/rehashing-the-src-layout/ + +.. _editable install: https://pip.pypa.io/en/stable/cli/pip_install/#editable-installs diff --git a/setuptools/__init__.py b/setuptools/__init__.py index 06991b65d7..15b1786e88 100644 --- a/setuptools/__init__.py +++ b/setuptools/__init__.py @@ -1,6 +1,5 @@ """Extensions to the 'distutils' for large or complex distributions""" -from fnmatch import fnmatchcase import functools import os import re @@ -9,7 +8,6 @@ import distutils.core from distutils.errors import DistutilsOptionError -from distutils.util import convert_path from ._deprecation_warning import SetuptoolsDeprecationWarning @@ -17,6 +15,7 @@ from setuptools.extension import Extension from setuptools.dist import Distribution from setuptools.depends import Require +from setuptools.discovery import PackageFinder, PEP420PackageFinder from . import monkey from . import logging @@ -37,85 +36,6 @@ bootstrap_install_from = None -class PackageFinder: - """ - Generate a list of all Python packages found within a directory - """ - - @classmethod - def find(cls, where='.', exclude=(), include=('*',)): - """Return a list all Python packages found within directory 'where' - - 'where' is the root directory which will be searched for packages. It - should be supplied as a "cross-platform" (i.e. URL-style) path; it will - be converted to the appropriate local path syntax. - - 'exclude' is a sequence of package names to exclude; '*' can be used - as a wildcard in the names, such that 'foo.*' will exclude all - subpackages of 'foo' (but not 'foo' itself). - - 'include' is a sequence of package names to include. If it's - specified, only the named packages will be included. If it's not - specified, all found packages will be included. 'include' can contain - shell style wildcard patterns just like 'exclude'. - """ - - return list( - cls._find_packages_iter( - convert_path(where), - cls._build_filter('ez_setup', '*__pycache__', *exclude), - cls._build_filter(*include), - ) - ) - - @classmethod - def _find_packages_iter(cls, where, exclude, include): - """ - All the packages found in 'where' that pass the 'include' filter, but - not the 'exclude' filter. - """ - for root, dirs, files in os.walk(where, followlinks=True): - # Copy dirs to iterate over it, then empty dirs. - all_dirs = dirs[:] - dirs[:] = [] - - for dir in all_dirs: - full_path = os.path.join(root, dir) - rel_path = os.path.relpath(full_path, where) - package = rel_path.replace(os.path.sep, '.') - - # Skip directory trees that are not valid packages - if '.' in dir or not cls._looks_like_package(full_path): - continue - - # Should this package be included? - if include(package) and not exclude(package): - yield package - - # Keep searching subdirectories, as there may be more packages - # down there, even if the parent was excluded. - dirs.append(dir) - - @staticmethod - def _looks_like_package(path): - """Does a directory look like a package?""" - return os.path.isfile(os.path.join(path, '__init__.py')) - - @staticmethod - def _build_filter(*patterns): - """ - Given a list of patterns, return a callable that will be true only if - the input matches at least one of the patterns. - """ - return lambda name: any(fnmatchcase(name, pat=pat) for pat in patterns) - - -class PEP420PackageFinder(PackageFinder): - @staticmethod - def _looks_like_package(path): - return True - - find_packages = PackageFinder.find find_namespace_packages = PEP420PackageFinder.find diff --git a/setuptools/discovery.py b/setuptools/discovery.py new file mode 100644 index 0000000000..2eb1f5ea76 --- /dev/null +++ b/setuptools/discovery.py @@ -0,0 +1,399 @@ +"""Automatic discovery of Python modules and packages (for inclusion in the +distribution) and other config values. + +For the purposes of this module, the following nomenclature is used: + +- "src-layout": a directory representing a Python project that contains a "src" + folder. Everything under the "src" folder is meant to be included in the + distribution when packaging the project. Example:: + + . + ├── tox.ini + ├── pyproject.toml + └── src/ + └── mypkg/ + ├── __init__.py + ├── mymodule.py + └── my_data_file.txt + +- "flat-layout": a Python project that does not use "src-layout" but instead + have a directory under the project root for each package:: + + . + ├── tox.ini + ├── pyproject.toml + └── mypkg/ + ├── __init__.py + ├── mymodule.py + └── my_data_file.txt + +- "single-module": a project that contains a single Python script direct under + the project root (no directory used):: + + . + ├── tox.ini + ├── pyproject.toml + └── mymodule.py + +""" + +import itertools +import os +from fnmatch import fnmatchcase +from glob import glob + +import _distutils_hack.override # noqa: F401 + +from distutils import log +from distutils.util import convert_path + +chain_iter = itertools.chain.from_iterable + + +def _valid_name(path): + # Ignore invalid names that cannot be imported directly + return os.path.basename(path).isidentifier() + + +class _Finder: + """Base class that exposes functionality for module/package finders""" + + ALWAYS_EXCLUDE = () + DEFAULT_EXCLUDE = () + + @classmethod + def find(cls, where='.', exclude=(), include=('*',)): + """Return a list of all Python items (packages or modules, depending on + the finder implementation) found within directory 'where'. + + 'where' is the root directory which will be searched. + It should be supplied as a "cross-platform" (i.e. URL-style) path; + it will be converted to the appropriate local path syntax. + + 'exclude' is a sequence of names to exclude; '*' can be used + as a wildcard in the names. + When finding packages, 'foo.*' will exclude all subpackages of 'foo' + (but not 'foo' itself). + + 'include' is a sequence of names to include. + If it's specified, only the named items will be included. + If it's not specified, all found items will be included. + 'include' can contain shell style wildcard patterns just like + 'exclude'. + """ + + exclude = exclude or cls.DEFAULT_EXCLUDE + return list( + cls._find_iter( + convert_path(where), + cls._build_filter(*cls.ALWAYS_EXCLUDE, *exclude), + cls._build_filter(*include), + ) + ) + + @classmethod + def _find_iter(cls, where, exclude, include): + raise NotImplementedError + + @staticmethod + def _build_filter(*patterns): + """ + Given a list of patterns, return a callable that will be true only if + the input matches at least one of the patterns. + """ + return lambda name: any(fnmatchcase(name, pat) for pat in patterns) + + +class PackageFinder(_Finder): + """ + Generate a list of all Python packages found within a directory + """ + + ALWAYS_EXCLUDE = ("ez_setup", "*__pycache__") + + @classmethod + def _find_iter(cls, where, exclude, include): + """ + All the packages found in 'where' that pass the 'include' filter, but + not the 'exclude' filter. + """ + for root, dirs, files in os.walk(where, followlinks=True): + # Copy dirs to iterate over it, then empty dirs. + all_dirs = dirs[:] + dirs[:] = [] + + for dir in all_dirs: + full_path = os.path.join(root, dir) + rel_path = os.path.relpath(full_path, where) + package = rel_path.replace(os.path.sep, '.') + + # Skip directory trees that are not valid packages + if '.' in dir or not cls._looks_like_package(full_path): + continue + + # Should this package be included? + if include(package) and not exclude(package): + yield package + + # Keep searching subdirectories, as there may be more packages + # down there, even if the parent was excluded. + dirs.append(dir) + + @staticmethod + def _looks_like_package(path): + """Does a directory look like a package?""" + return os.path.isfile(os.path.join(path, '__init__.py')) + + +class PEP420PackageFinder(PackageFinder): + @staticmethod + def _looks_like_package(path): + return True + + +class ModuleFinder(_Finder): + """Find isolated Python modules. + This function will **not** recurse subdirectories. + """ + + @classmethod + def _find_iter(cls, where, exclude, include): + for file in glob(os.path.join(where, "*.py")): + module, _ext = os.path.splitext(os.path.basename(file)) + + if not cls._looks_like_module(module): + continue + + if include(module) and not exclude(module): + yield module + + _looks_like_module = staticmethod(_valid_name) + + +# We have to be extra careful in the case of flat layout to not include files +# and directories not meant for distribution (e.g. tool-related) + + +class FlatLayoutPackageFinder(PEP420PackageFinder): + _EXCLUDE = ( + "bin", + "doc", + "docs", + "documentation", + "test", + "tests", + "example", + "examples", + "scripts", + "tools", + "build", + "dist", + "venv", + # ---- Task runners / Build tools ---- + "tasks", # invoke + "fabfile", # fabric + "site_scons", # SCons + # ---- Hidden directories/Private packages ---- + "[._]*", + ) + + DEFAULT_EXCLUDE = tuple(chain_iter((p, f"{p}.*") for p in _EXCLUDE)) + """Reserved package names""" + + _looks_like_package = staticmethod(_valid_name) + + +class FlatLayoutModuleFinder(ModuleFinder): + DEFAULT_EXCLUDE = ( + "setup", + "conftest", + "test", + "tests", + "example", + "examples", + "build", + # ---- Task runners ---- + "noxfile", + "pavement", + "dodo", + "tasks", + "fabfile", + # ---- Other tools ---- + "[Ss][Cc]onstruct", # SCons + "conanfile", # Connan: C/C++ build tool + "manage", # Django + # ---- Hidden files/Private modules ---- + "[._]*", + ) + """Reserved top-level module names""" + + +def _find_packages_within(root_pkg, pkg_dir): + nested = PEP420PackageFinder.find(pkg_dir) + return [root_pkg] + [".".join((root_pkg, n)) for n in nested] + + +class ConfigDiscovery: + """Fill-in metadata and options that can be automatically derived + (from other metadata/options, the file system or conventions) + """ + + def __init__(self, distribution): + self.dist = distribution + self._called = False + self._root_dir = None # delay so `src_root` can be set in dist + + def __call__(self, force=False): + """Automatically discover missing configuration fields + and modifies the given ``distribution`` object in-place. + + Note that by default this will only have an effect the first time the + ``ConfigDiscovery`` object is called. + + To repeatedly invoke automatic discovery (e.g. when the project + directory changes), please use ``force=True`` (or create a new + ``ConfigDiscovery`` instance). + """ + if force is False and self._called: + # Avoid overhead of multiple calls + return + + self._root_dir = self.dist.src_root or os.curdir + + self._analyse_package_layout() + self._analyse_name() # depends on ``packages`` and ``py_modules`` + + self._called = True + + def _analyse_package_layout(self): + if self.dist.packages is not None or self.dist.py_modules is not None: + # For backward compatibility, just try to find modules/packages + # when nothing is given + return None + + log.debug( + "No `packages` or `py_modules` configuration, performing " + "automatic discovery." + ) + + return ( + self._analyse_explicit_layout() + or self._analyse_src_layout() + # flat-layout is the trickiest for discovery so it should be last + or self._analyse_flat_layout() + ) + + def _analyse_explicit_layout(self): + """The user can explicitly give a package layout via ``package_dir``""" + package_dir = (self.dist.package_dir or {}).copy() + package_dir.pop("", None) # This falls under the "src-layout" umbrella + root_dir = self._root_dir + + if not package_dir: + return False + + pkgs = chain_iter( + _find_packages_within(pkg, os.path.join(root_dir, parent_dir)) + for pkg, parent_dir in package_dir.items() + ) + self.dist.packages = list(pkgs) + log.debug(f"`explicit-layout` detected -- analysing {package_dir}") + return True + + def _analyse_src_layout(self): + """Try to find all packages or modules under the ``src`` directory + (or anything pointed by ``package_dir[""]``). + + The "src-layout" is relatively safe for automatic discovery. + We assume that everything within is meant to be included in the + distribution. + + If ``package_dir[""]`` is not given, but the ``src`` directory exists, + this function will set ``package_dir[""] = "src"``. + """ + package_dir = self.dist.package_dir = self.dist.package_dir or {} + src_dir = os.path.join(self._root_dir, package_dir.get("", "src")) + if not os.path.isdir(src_dir): + return False + + package_dir.setdefault("", os.path.basename(src_dir)) + self.dist.packages = PEP420PackageFinder.find(src_dir) + self.dist.py_modules = ModuleFinder.find(src_dir) + log.debug(f"`src-layout` detected -- analysing {src_dir}") + return True + + def _analyse_flat_layout(self): + """Try to find all packages and modules under the project root""" + self.dist.packages = FlatLayoutPackageFinder.find(self._root_dir) + self.dist.py_modules = FlatLayoutModuleFinder.find(self._root_dir) + log.debug(f"`flat-layout` detected -- analysing {self._root_dir}") + return True + + def _analyse_name(self): + """The packages/modules are the essential contribution of the author. + Therefore the name of the distribution can be derived from them. + """ + if self.dist.metadata.name or self.dist.name: + # get_name() is not reliable (can return "UNKNOWN") + return None + + log.debug("No `name` configuration, performing automatic discovery") + + name = ( + self._find_name_single_package_or_module() + or self._find_name_from_packages() + ) + if name: + self.dist.metadata.name = name + self.dist.name = name + + def _find_name_single_package_or_module(self): + """Exactly one module or package""" + for field in ('packages', 'py_modules'): + items = getattr(self.dist, field, None) or [] + if items and len(items) == 1: + log.debug(f"Single module/package detected, name: {items[0]}") + return items[0] + + return None + + def _find_name_from_packages(self): + """Try to find the root package that is not a PEP 420 namespace""" + if not self.dist.packages: + return None + + packages = sorted(self.dist.packages, key=len) + common_ancestors = [] + for i, name in enumerate(packages): + if not all(n.startswith(name) for n in packages[i+1:]): + # Since packages are sorted by length, this condition is able + # to find a list of all common ancestors. + # When there is divergence (e.g. multiple root packages) + # the list will be empty + break + common_ancestors.append(name) + + for name in common_ancestors: + init = os.path.join(self._find_package_path(name), "__init__.py") + if os.path.isfile(init): + log.debug(f"Common parent package detected, name: {name}") + return name + + log.warn("No parent package detected, impossible to derive `name`") + return None + + def _find_package_path(self, name): + """Given a package name, return the path where it should be found on + disk, considering the ``package_dir`` option. + """ + package_dir = self.dist.package_dir or {} + parts = name.split(".") + for i in range(len(parts), 0, -1): + # Look backwards, the most specific package_dir first + partial_name = ".".join(parts[:i]) + if partial_name in package_dir: + parent = package_dir[partial_name] + return os.path.join(self._root_dir, parent, *parts[i:]) + + parent = (package_dir.get("") or "").split("/") + return os.path.join(self._root_dir, *parent, *parts) diff --git a/setuptools/dist.py b/setuptools/dist.py index c0e8e1b31a..4743eeedb4 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -40,6 +40,8 @@ from setuptools import windows_support from setuptools.monkey import get_unpatched from setuptools.config import setupcfg, pyprojecttoml +from setuptools.discovery import ConfigDiscovery + import pkg_resources from setuptools.extern.packaging import version, requirements from . import _reqs @@ -465,6 +467,8 @@ def __init__(self, attrs=None): }, ) + self.set_defaults = ConfigDiscovery(self) + self._set_metadata_defaults(attrs) self.metadata.version = self._normalize_version( @@ -1196,6 +1200,13 @@ def handle_display_options(self, option_order): sys.stdout.detach(), encoding, errors, newline, line_buffering ) + def run_command(self, command): + self.set_defaults() + # Postpone defaults until all explicit configuration is considered + # (setup() args, config files, command line and plugins) + + super().run_command(command) + class DistDeprecationWarning(SetuptoolsDeprecationWarning): """Class for warning about deprecations in dist in diff --git a/setuptools/tests/integration/helpers.py b/setuptools/tests/integration/helpers.py index 43f43902e3..24c02be042 100644 --- a/setuptools/tests/integration/helpers.py +++ b/setuptools/tests/integration/helpers.py @@ -8,6 +8,7 @@ import subprocess import tarfile from zipfile import ZipFile +from pathlib import Path def run(cmd, env=None): @@ -59,3 +60,16 @@ def get_content(self, zip_or_tar_info): raise ValueError(msg) return str(content.read(), "utf-8") return str(self._obj.read(zip_or_tar_info), "utf-8") + + +def get_sdist_members(sdist_path): + with tarfile.open(sdist_path, "r:gz") as tar: + files = [Path(f) for f in tar.getnames()] + # remove root folder + relative_files = ("/".join(f.parts[1:]) for f in files) + return {f for f in relative_files if f} + + +def get_wheel_members(wheel_path): + with ZipFile(wheel_path) as zipfile: + return set(zipfile.namelist()) diff --git a/setuptools/tests/test_config_discovery.py b/setuptools/tests/test_config_discovery.py new file mode 100644 index 0000000000..2215cddb86 --- /dev/null +++ b/setuptools/tests/test_config_discovery.py @@ -0,0 +1,222 @@ +import os +import sys +from configparser import ConfigParser +from itertools import product + +from setuptools.command.sdist import sdist +from setuptools.dist import Distribution + +import pytest + +from .contexts import quiet +from .integration.helpers import get_sdist_members, get_wheel_members, run +from .textwrap import DALS + + +class TestDiscoverPackagesAndPyModules: + """Make sure discovered values for ``packages`` and ``py_modules`` work + similarly to explicit configuration for the simple scenarios. + """ + OPTIONS = { + # Different options according to the circumstance being tested + "explicit-src": { + "package_dir": {"": "src"}, + "packages": ["pkg"] + }, + "variation-lib": { + "package_dir": {"": "lib"}, # variation of the source-layout + }, + "explicit-flat": { + "packages": ["pkg"] + }, + "explicit-single_module": { + "py_modules": ["pkg"] + }, + "explicit-namespace": { + "packages": ["ns", "ns.pkg"] + }, + "automatic-src": {}, + "automatic-flat": {}, + "automatic-single_module": {}, + "automatic-namespace": {} + } + FILES = { + "src": ["src/pkg/__init__.py", "src/pkg/main.py"], + "lib": ["lib/pkg/__init__.py", "lib/pkg/main.py"], + "flat": ["pkg/__init__.py", "pkg/main.py"], + "single_module": ["pkg.py"], + "namespace": ["ns/pkg/__init__.py"] + } + + def _get_info(self, circumstance): + _, _, layout = circumstance.partition("-") + files = self.FILES[layout] + options = self.OPTIONS[circumstance] + return files, options + + @pytest.mark.parametrize("circumstance", OPTIONS.keys()) + def test_sdist_filelist(self, tmp_path, circumstance): + files, options = self._get_info(circumstance) + _populate_project_dir(tmp_path, files, options) + + here = os.getcwd() + root = "/".join(os.path.split(tmp_path)) # POSIX-style + dist = Distribution({**options, "src_root": root}) + dist.script_name = 'setup.py' + dist.set_defaults() + cmd = sdist(dist) + cmd.ensure_finalized() + assert cmd.distribution.packages or cmd.distribution.py_modules + + with quiet(): + try: + os.chdir(tmp_path) + cmd.run() + finally: + os.chdir(here) + + manifest = [f.replace(os.sep, "/") for f in cmd.filelist.files] + for file in files: + assert any(f.endswith(file) for f in manifest) + + @pytest.mark.parametrize("circumstance", OPTIONS.keys()) + def test_project(self, tmp_path, circumstance): + files, options = self._get_info(circumstance) + _populate_project_dir(tmp_path, files, options) + + # Simulate a pre-existing `build` directory + (tmp_path / "build").mkdir() + (tmp_path / "build/lib").mkdir() + (tmp_path / "build/bdist.linux-x86_64").mkdir() + (tmp_path / "build/bdist.linux-x86_64/file.py").touch() + (tmp_path / "build/lib/__init__.py").touch() + (tmp_path / "build/lib/file.py").touch() + (tmp_path / "dist").mkdir() + (tmp_path / "dist/file.py").touch() + + _run_build(tmp_path) + + sdist_files = get_sdist_members(next(tmp_path.glob("dist/*.tar.gz"))) + print("~~~~~ sdist_members ~~~~~") + print('\n'.join(sdist_files)) + assert sdist_files >= set(files) + + wheel_files = get_wheel_members(next(tmp_path.glob("dist/*.whl"))) + print("~~~~~ wheel_members ~~~~~") + print('\n'.join(wheel_files)) + orig_files = {f.replace("src/", "").replace("lib/", "") for f in files} + assert wheel_files >= orig_files + + # Make sure build files are not included by mistake + for file in wheel_files: + assert "build" not in files + assert "dist" not in files + + PURPOSEFULLY_EMPY = { + "setup.cfg": DALS( + """ + [metadata] + name = myproj + version = 0.0.0 + + [options] + {param} = + """ + ), + "setup.py": DALS( + """ + __import__('setuptools').setup( + name="myproj", + version="0.0.0", + {param}=[] + ) + """ + ), + "pyproject.toml": DALS( + """ + [build-system] + requires = [] + build-backend = 'setuptools.build_meta' + """ + ) + } + + @pytest.mark.parametrize( + "config_file, param, circumstance", + product(["setup.cfg", "setup.py"], ["packages", "py_modules"], FILES.keys()) + ) + def test_purposefully_empty(self, tmp_path, config_file, param, circumstance): + files = self.FILES[circumstance] + _populate_project_dir(tmp_path, files, {}) + config = self.PURPOSEFULLY_EMPY[config_file].format(param=param) + (tmp_path / config_file).write_text(config) + + # Make sure build works with or without setup.cfg + pyproject = self.PURPOSEFULLY_EMPY["pyproject.toml"] + (tmp_path / "pyproject.toml").write_text(pyproject) + + _run_build(tmp_path) + + wheel_files = get_wheel_members(next(tmp_path.glob("dist/*.whl"))) + print("~~~~~ wheel_members ~~~~~") + print('\n'.join(wheel_files)) + for file in files: + name = file.replace("src/", "") + assert name not in wheel_files + + +class TestNoConfig: + DEFAULT_VERSION = "0.0.0" # Default version given by setuptools + + EXAMPLES = { + "pkg1": ["src/pkg1.py"], + "pkg2": ["src/pkg2/__init__.py"], + "ns.nested.pkg3": ["src/ns/nested/pkg3/__init__.py"] + } + + @pytest.mark.parametrize("example", EXAMPLES.keys()) + def test_discover_name(self, tmp_path, example): + _populate_project_dir(tmp_path, self.EXAMPLES[example], {}) + _run_build(tmp_path, "--sdist") + # Expected distribution file + dist_file = tmp_path / f"dist/{example}-{self.DEFAULT_VERSION}.tar.gz" + assert dist_file.is_file() + + +def _populate_project_dir(root, files, options): + # NOTE: Currently pypa/build will refuse to build the project if no + # `pyproject.toml` or `setup.py` is found. So it is impossible to do + # completely "config-less" projects. + (root / "setup.py").write_text("import setuptools\nsetuptools.setup()") + (root / "README.md").write_text("# Example Package") + (root / "LICENSE").write_text("Copyright (c) 2018") + _write_setupcfg(root, options) + paths = (root / f for f in files) + for path in paths: + path.parent.mkdir(exist_ok=True, parents=True) + path.touch() + + +def _write_setupcfg(root, options): + if not options: + print("~~~~~ **NO** setup.cfg ~~~~~") + return + setupcfg = ConfigParser() + setupcfg.add_section("options") + for key, value in options.items(): + if isinstance(value, list): + setupcfg["options"][key] = ", ".join(value) + elif isinstance(value, dict): + str_value = "\n".join(f"\t{k} = {v}" for k, v in value.items()) + setupcfg["options"][key] = "\n" + str_value + else: + setupcfg["options"][key] = str(value) + with open(root / "setup.cfg", "w") as f: + setupcfg.write(f) + print("~~~~~ setup.cfg ~~~~~") + print((root / "setup.cfg").read_text()) + + +def _run_build(path, *flags): + cmd = [sys.executable, "-m", "build", "--no-isolation", *flags, str(path)] + return run(cmd, env={'DISTUTILS_DEBUG': '1'}) diff --git a/setuptools/tests/test_dist.py b/setuptools/tests/test_dist.py index 4980f2c3ce..049576a732 100644 --- a/setuptools/tests/test_dist.py +++ b/setuptools/tests/test_dist.py @@ -2,6 +2,7 @@ import collections import re import functools +import os import urllib.request import urllib.parse from distutils.errors import DistutilsSetupError @@ -18,6 +19,7 @@ from .textwrap import DALS from .test_easy_install import make_nspkg_sdist +from .test_find_packages import ensure_files import pytest @@ -69,16 +71,19 @@ def test_dist__get_unpatched_deprecated(): pytest.warns(DistDeprecationWarning, _get_unpatched, [""]) +EXAMPLE_BASE_INFO = dict( + name="package", + version="0.0.1", + author="Foo Bar", + author_email="foo@bar.net", + long_description="Long\ndescription", + description="Short description", + keywords=["one", "two"], +) + + def __read_test_cases(): - base = dict( - name="package", - version="0.0.1", - author="Foo Bar", - author_email="foo@bar.net", - long_description="Long\ndescription", - description="Short description", - keywords=["one", "two"], - ) + base = EXAMPLE_BASE_INFO params = functools.partial(dict, base) @@ -379,3 +384,127 @@ def test_rfc822_unescape(content, result): def test_metadata_name(): with pytest.raises(DistutilsSetupError, match='missing.*name'): Distribution()._validate_metadata() + + +@pytest.mark.parametrize( + "dist_name, py_module", + [ + ("my.pkg", "my_pkg"), + ("my-pkg", "my_pkg"), + ("my_pkg", "my_pkg"), + ("pkg", "pkg"), + ] +) +def test_dist_default_py_modules(tmp_path, dist_name, py_module): + (tmp_path / f"{py_module}.py").touch() + + (tmp_path / "setup.py").touch() + (tmp_path / "noxfile.py").touch() + # ^-- make sure common tool files are ignored + + attrs = { + **EXAMPLE_BASE_INFO, + "name": dist_name, + "src_root": str(tmp_path) + } + # Find `py_modules` corresponding to dist_name if not given + dist = Distribution(attrs) + dist.set_defaults() + assert dist.py_modules == [py_module] + # When `py_modules` is given, don't do anything + dist = Distribution({**attrs, "py_modules": ["explicity_py_module"]}) + dist.set_defaults() + assert dist.py_modules == ["explicity_py_module"] + # When `packages` is given, don't do anything + dist = Distribution({**attrs, "packages": ["explicity_package"]}) + dist.set_defaults() + assert not dist.py_modules + + +@pytest.mark.parametrize( + "dist_name, package_dir, package_files, packages", + [ + ("my.pkg", None, ["my_pkg/__init__.py", "my_pkg/mod.py"], ["my_pkg"]), + ("my-pkg", None, ["my_pkg/__init__.py", "my_pkg/mod.py"], ["my_pkg"]), + ("my_pkg", None, ["my_pkg/__init__.py", "my_pkg/mod.py"], ["my_pkg"]), + ("my.pkg", None, ["my/pkg/__init__.py"], ["my", "my.pkg"]), + ( + "my_pkg", + None, + ["src/my_pkg/__init__.py", "src/my_pkg2/__init__.py"], + ["my_pkg", "my_pkg2"] + ), + ( + "my_pkg", + {"pkg": "lib", "pkg2": "lib2"}, + ["lib/__init__.py", "lib/nested/__init__.pyt", "lib2/__init__.py"], + ["pkg", "pkg.nested", "pkg2"] + ), + ] +) +def test_dist_default_packages( + tmp_path, dist_name, package_dir, package_files, packages +): + ensure_files(tmp_path, package_files) + + (tmp_path / "setup.py").touch() + (tmp_path / "noxfile.py").touch() + # ^-- should not be included by default + + attrs = { + **EXAMPLE_BASE_INFO, + "name": dist_name, + "src_root": str(tmp_path), + "package_dir": package_dir + } + # Find `packages` either corresponding to dist_name or inside src + dist = Distribution(attrs) + dist.set_defaults() + assert not dist.py_modules + assert not dist.py_modules + assert set(dist.packages) == set(packages) + # When `py_modules` is given, don't do anything + dist = Distribution({**attrs, "py_modules": ["explicit_py_module"]}) + dist.set_defaults() + assert not dist.packages + assert set(dist.py_modules) == {"explicit_py_module"} + # When `packages` is given, don't do anything + dist = Distribution({**attrs, "packages": ["explicit_package"]}) + dist.set_defaults() + assert not dist.py_modules + assert set(dist.packages) == {"explicit_package"} + + +@pytest.mark.parametrize( + "dist_name, package_dir, package_files", + [ + ("my.pkg.nested", None, ["my/pkg/nested/__init__.py"]), + ("my.pkg", None, ["my/pkg/__init__.py", "my/pkg/file.py"]), + ("my_pkg", None, ["my_pkg.py"]), + ("my_pkg", None, ["my_pkg/__init__.py", "my_pkg/nested/__init__.py"]), + ("my_pkg", None, ["src/my_pkg/__init__.py", "src/my_pkg/nested/__init__.py"]), + ( + "my_pkg", + {"my_pkg": "lib", "my_pkg.lib2": "lib2"}, + ["lib/__init__.py", "lib/nested/__init__.pyt", "lib2/__init__.py"], + ), + # Should not try to guess a name from multiple py_modules/packages + ("UNKNOWN", None, ["mod1.py", "mod2.py"]), + ("UNKNOWN", None, ["pkg1/__ini__.py", "pkg2/__init__.py"]), + ("UNKNOWN", None, ["src/pkg1/__ini__.py", "src/pkg2/__init__.py"]), + ] +) +def test_dist_default_name(tmp_path, dist_name, package_dir, package_files): + """Make sure dist.name is discovered from packages/py_modules""" + ensure_files(tmp_path, package_files) + attrs = { + **EXAMPLE_BASE_INFO, + "src_root": "/".join(os.path.split(tmp_path)), # POSIX-style + "package_dir": package_dir + } + del attrs["name"] + + dist = Distribution(attrs) + dist.set_defaults() + assert dist.py_modules or dist.packages + assert dist.get_name() == dist_name diff --git a/setuptools/tests/test_find_packages.py b/setuptools/tests/test_find_packages.py index 906713f61d..efcce924e5 100644 --- a/setuptools/tests/test_find_packages.py +++ b/setuptools/tests/test_find_packages.py @@ -1,4 +1,4 @@ -"""Tests for setuptools.find_packages().""" +"""Tests for automatic package discovery""" import os import sys import shutil @@ -9,6 +9,7 @@ from setuptools import find_packages from setuptools import find_namespace_packages +from setuptools.discovery import FlatLayoutPackageFinder # modeled after CPython's test.support.can_symlink @@ -178,3 +179,67 @@ def test_pep420_ns_package_no_non_package_dirs(self): shutil.rmtree(os.path.join(self.dist_dir, 'pkg/subpkg/assets')) packages = find_namespace_packages(self.dist_dir) self._assert_packages(packages, ['pkg', 'pkg.nspkg', 'pkg.subpkg']) + + +class TestFlatLayoutPackageFinder: + EXAMPLES = { + "hidden-folders": ( + [".pkg/__init__.py", "pkg/__init__.py", "pkg/nested/file.txt"], + ["pkg", "pkg.nested"] + ), + "private-packages": ( + ["_pkg/__init__.py", "pkg/_private/__init__.py"], + ["pkg", "pkg._private"] + ), + "invalid-name": ( + ["invalid-pkg/__init__.py", "other.pkg/__init__.py", "yet,another/file.py"], + [] + ), + "docs": ( + ["pkg/__init__.py", "docs/conf.py", "docs/readme.rst"], + ["pkg"] + ), + "tests": ( + ["pkg/__init__.py", "tests/test_pkg.py", "tests/__init__.py"], + ["pkg"] + ), + "examples": ( + [ + "pkg/__init__.py", + "examples/__init__.py", + "examples/file.py" + "example/other_file.py", + # Sub-packages should always be fine + "pkg/example/__init__.py", + "pkg/examples/__init__.py", + ], + ["pkg", "pkg.examples", "pkg.example"] + ), + "tool-specific": ( + [ + "pkg/__init__.py", + "tasks/__init__.py", + "tasks/subpackage/__init__.py", + "fabfile/__init__.py", + "fabfile/subpackage/__init__.py", + # Sub-packages should always be fine + "pkg/tasks/__init__.py", + "pkg/fabfile/__init__.py", + ], + ["pkg", "pkg.tasks", "pkg.fabfile"] + ) + } + + @pytest.mark.parametrize("example", EXAMPLES.keys()) + def test_unwanted_directories_not_included(self, tmp_path, example): + files, expected_packages = self.EXAMPLES[example] + ensure_files(tmp_path, files) + found_packages = FlatLayoutPackageFinder.find(str(tmp_path)) + assert set(found_packages) == set(expected_packages) + + +def ensure_files(root_path, files): + for file in files: + path = root_path / file + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() diff --git a/setuptools/tests/test_find_py_modules.py b/setuptools/tests/test_find_py_modules.py new file mode 100644 index 0000000000..4ef6880120 --- /dev/null +++ b/setuptools/tests/test_find_py_modules.py @@ -0,0 +1,81 @@ +"""Tests for automatic discovery of modules""" +import os + +import pytest + +from setuptools.discovery import FlatLayoutModuleFinder, ModuleFinder + +from .test_find_packages import ensure_files, has_symlink + + +class TestModuleFinder: + def find(self, path, *args, **kwargs): + return set(ModuleFinder.find(str(path), *args, **kwargs)) + + EXAMPLES = { + # circumstance: (files, kwargs, expected_modules) + "simple_folder": ( + ["file.py", "other.py"], + {}, # kwargs + ["file", "other"], + ), + "exclude": ( + ["file.py", "other.py"], + {"exclude": ["f*"]}, + ["other"], + ), + "include": ( + ["file.py", "fole.py", "other.py"], + {"include": ["f*"], "exclude": ["fo*"]}, + ["file"], + ), + "invalid-name": ( + ["my-file.py", "other.file.py"], + {}, + [] + ) + } + + @pytest.mark.parametrize("example", EXAMPLES.keys()) + def test_finder(self, tmp_path, example): + files, kwargs, expected_modules = self.EXAMPLES[example] + ensure_files(tmp_path, files) + assert self.find(tmp_path, **kwargs) == set(expected_modules) + + @pytest.mark.skipif(not has_symlink(), reason='Symlink support required') + def test_symlinked_packages_are_included(self, tmp_path): + src = "_myfiles/file.py" + ensure_files(tmp_path, [src]) + os.symlink(tmp_path / src, tmp_path / "link.py") + assert self.find(tmp_path) == {"link"} + + +class TestFlatLayoutModuleFinder: + def find(self, path, *args, **kwargs): + return set(FlatLayoutModuleFinder.find(str(path))) + + EXAMPLES = { + # circumstance: (files, expected_modules) + "hidden-files": ( + [".module.py"], + [] + ), + "private-modules": ( + ["_module.py"], + [] + ), + "common-names": ( + ["setup.py", "conftest.py", "test.py", "tests.py", "example.py", "mod.py"], + ["mod"] + ), + "tool-specific": ( + ["tasks.py", "fabfile.py", "noxfile.py", "dodo.py", "manage.py", "mod.py"], + ["mod"] + ) + } + + @pytest.mark.parametrize("example", EXAMPLES.keys()) + def test_unwanted_files_not_included(self, tmp_path, example): + files, expected_modules = self.EXAMPLES[example] + ensure_files(tmp_path, files) + assert self.find(tmp_path) == set(expected_modules)