Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add default values for the packages and py_modules options #2888

Closed
wants to merge 7 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions changelog.d/2887.change.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
Added default values for the ``py_modules`` and ``packages`` options.
Setuptools will try to find the best values assuming that the package uses
either the *src-layout* (a ``src`` directory containing all the packages),
the *flat-layout* (a single directory with the same name as the distribution
as the top-level package), or the *single-module* approach (a single Python
file with the same name as the distribution).
This behavior will be observed **only if both of options are not explicitly
set**.
2 changes: 2 additions & 0 deletions setup.cfg
Original file line number Diff line number Diff line change
@@ -64,6 +64,8 @@ testing =
sphinx
jaraco.path>=3.2.0

build

docs =
# upstream
sphinx
98 changes: 98 additions & 0 deletions setuptools/dist.py
Original file line number Diff line number Diff line change
@@ -30,6 +30,7 @@
from setuptools.extern import packaging
from setuptools.extern import ordered_set
from setuptools.extern.more_itertools import unique_everseen
from setuptools.extern.packaging import utils

from . import SetuptoolsDeprecationWarning

@@ -359,6 +360,11 @@ def check_packages(dist, attr, value):
)


def _find_packages_within(root_pkg, pkg_dir):
nested = setuptools.find_namespace_packages(pkg_dir)
return [root_pkg] + [".".join((root_pkg, n)) for n in nested]


_Distribution = get_unpatched(distutils.core.Distribution)


@@ -459,6 +465,8 @@ def __init__(self, attrs=None):
},
)

self._option_defaults_already_set = False

self._set_metadata_defaults(attrs)

self.metadata.version = self._normalize_version(
@@ -476,6 +484,88 @@ def _set_metadata_defaults(self, attrs):
for option, default in self._DISTUTILS_UNSUPPORTED_METADATA.items():
vars(self.metadata).setdefault(option, attrs.get(option, default()))

def set_option_defaults(self):
"""Fill-in options that can be automatically derived
(from other options, the file system itself and conventions).
"""
if self.packages or self.py_modules:
# For backward compatibility, just try to find modules/packages
# when nothing is given
return

if not self.packages:
self.packages = self._default_packages()

if not self.packages:
# If packages are found there is no need to consider single files
self.py_modules = self._default_py_modules()

self._option_defaults_already_set = True

def _default_py_modules(self):
"""By default setuptools will try to find a single Python module in the
root directory of the project whose name matches the distribution name.
This covers the common use case of single module packages.
"""
root_dir = self.src_root or os.getcwd()

if not self.metadata.name:
return []

canonical_name = utils.canonicalize_name(self.metadata.name)
module_name = canonical_name.replace("-", "_")

# Single module package
single_module = os.path.join(root_dir, module_name + ".py")
if os.path.isfile(single_module):
return [module_name]

return []

def _default_packages(self):
"""By default setuptools will try to find a directory that matches the
distribution name (or at least the first part of it in the case of
``.`` separated namespaces). If that directory exists, it will be
considered the top-level package of the distribution (this covers the
use case know as *flat-layout*).
When this directory does not exists, it will try to find packages under
``src``, or anything pointed by ``package_dir[""]`` (this covers the
use case know as *src-layout*).
"""
root_dir = self.src_root or os.getcwd()

# ---- Simple scenario, specific package_dir is given ----
if self.package_dir and "" not in self.package_dir:
pkgs = itertools.chain.from_iterable(
_find_packages_within(pkg, os.path.join(root_dir, dirname))
for pkg, dirname in (self.package_dir or {}).items()
)
return list(pkgs)

# ---- "flat" layout: single folder with package name ----
if self.metadata.name:
canonical_name = utils.canonicalize_name(self.metadata.name)
package_name = canonical_name.replace("-", "_")
# namespaces are indicated with "." so we cannot use canonical_name
namespaced = re.sub(r"\.+", ".", self.metadata.name)
namespaced = re.sub(r"[-_]+", "_", namespaced)
namespace, _, _ = namespaced.partition(".")
for candidate in (package_name, namespace):
pkg_dir = os.path.join(root_dir, candidate)
if os.path.isdir(pkg_dir):
return _find_packages_within(candidate, pkg_dir)

# ---- "src" layout: single folder with package name ----
self.package_dir = self.package_dir or {}
src_dir = os.path.join(root_dir, self.package_dir.get("", "src"))
if not os.path.isdir(src_dir):
return []

self.package_dir.setdefault("", os.path.basename(src_dir))
return setuptools.find_namespace_packages(src_dir)

@staticmethod
def _normalize_version(version):
if isinstance(version, setuptools.sic) or version is None:
@@ -1144,6 +1234,14 @@ def handle_display_options(self, option_order):
sys.stdout.detach(), encoding, errors, newline, line_buffering
)

def run_command(self, command):
if not self._option_defaults_already_set:
# Postpone default options until all configuration is considered
# (setup() args, config files, command line and plugins)
self.set_option_defaults()

super().run_command(command)


class DistDeprecationWarning(SetuptoolsDeprecationWarning):
"""Class for warning about deprecations in dist in
110 changes: 101 additions & 9 deletions setuptools/tests/test_dist.py
Original file line number Diff line number Diff line change
@@ -69,16 +69,19 @@ def test_dist__get_unpatched_deprecated():
pytest.warns(DistDeprecationWarning, _get_unpatched, [""])


EXAMPLE_BASE_INFO = dict(
name="package",
version="0.0.1",
author="Foo Bar",
author_email="foo@bar.net",
long_description="Long\ndescription",
description="Short description",
keywords=["one", "two"],
)


def __read_test_cases():
base = dict(
name="package",
version="0.0.1",
author="Foo Bar",
author_email="foo@bar.net",
long_description="Long\ndescription",
description="Short description",
keywords=["one", "two"],
)
base = EXAMPLE_BASE_INFO

params = functools.partial(dict, base)

@@ -374,3 +377,92 @@ def test_check_specifier():
)
def test_rfc822_unescape(content, result):
assert (result or content) == rfc822_unescape(rfc822_escape(content))


@pytest.mark.parametrize(
"dist_name, py_module",
[
("my.pkg", "my_pkg"),
("my-pkg", "my_pkg"),
("my_pkg", "my_pkg"),
("pkg", "pkg"),
]
)
def test_dist_default_py_modules(tmp_path, dist_name, py_module):
(tmp_path / f"{py_module}.py").touch()
(tmp_path / "otherfile.py").touch()
# ^-- just files matching dist name should be included by default

attrs = {
**EXAMPLE_BASE_INFO,
"name": dist_name,
"src_root": str(tmp_path)
}
# Find `py_modules` corresponding to dist_name if not given
dist = Distribution(attrs)
dist.set_option_defaults()
assert dist.py_modules == [py_module]
# When `py_modules` is given, don't do anything
dist = Distribution({**attrs, "py_modules": ["explicity_py_module"]})
dist.set_option_defaults()
assert dist.py_modules == ["explicity_py_module"]
# When `packages` is given, don't do anything
dist = Distribution({**attrs, "packages": ["explicity_package"]})
dist.set_option_defaults()
assert not dist.py_modules


@pytest.mark.parametrize(
"dist_name, package_dir, package_files, packages",
[
("my.pkg", None, ["my_pkg/__init__.py", "my_pkg/mod.py"], ["my_pkg"]),
("my-pkg", None, ["my_pkg/__init__.py", "my_pkg/mod.py"], ["my_pkg"]),
("my_pkg", None, ["my_pkg/__init__.py", "my_pkg/mod.py"], ["my_pkg"]),
("my.pkg", None, ["my/pkg/__init__.py"], ["my", "my.pkg"]),
(
"my_pkg",
None,
["src/my_pkg/__init__.py", "src/my_pkg2/__init__.py"],
["my_pkg", "my_pkg2"]
),
(
"my_pkg",
{"pkg": "lib", "pkg2": "lib2"},
["lib/__init__.py", "lib/nested/__init__.pyt", "lib2/__init__.py"],
["pkg", "pkg.nested", "pkg2"]
),
]
)
def test_dist_default_packages(
tmp_path, dist_name, package_dir, package_files, packages
):
for file in package_files:
path = tmp_path / file
path.parent.mkdir(parents=True, exist_ok=True)
path.touch()

(tmp_path / "otherfile.py").touch()
# ^-- should not be included by default

attrs = {
**EXAMPLE_BASE_INFO,
"name": dist_name,
"src_root": str(tmp_path),
"package_dir": package_dir
}
# Find `packages` either corresponding to dist_name or inside src
dist = Distribution(attrs)
dist.set_option_defaults()
assert not dist.py_modules
assert not dist.py_modules
assert set(dist.packages) == set(packages)
# When `py_modules` is given, don't do anything
dist = Distribution({**attrs, "py_modules": ["explicit_py_module"]})
dist.set_option_defaults()
assert not dist.packages
assert set(dist.py_modules) == {"explicit_py_module"}
# When `packages` is given, don't do anything
dist = Distribution({**attrs, "packages": ["explicit_package"]})
dist.set_option_defaults()
assert not dist.py_modules
assert set(dist.packages) == {"explicit_package"}
167 changes: 167 additions & 0 deletions setuptools/tests/test_option_defaults.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
import os
import subprocess
import sys
import tarfile
from configparser import ConfigParser
from pathlib import Path
from subprocess import CalledProcessError
from zipfile import ZipFile

import pytest

from setuptools.command.sdist import sdist
from setuptools.dist import Distribution

from .contexts import quiet


class TestDefaultPackagesAndPyModules:
"""Make sure default values for ``packages`` and ``py_modules`` work
similarly to explicit configuration for the simple scenarios.
"""

METADATA = {
"name": "example",
"version": "0.0.1",
"author": "Example Author"
}
OPTIONS = {
# Different options according to the circumstance being tested
"explicit-src": {
"package_dir": {"": "src"},
"packages": ["example"]
},
"explicit-flat": {
"packages": ["example"]
},
"explicit-single_module": {
"py_modules": ["example"]
},
"explicit-namespace": {
"packages": ["ns", "ns.example"]
},
"automatic-src": {},
"automatic-flat": {},
"automatic-single_module": {},
"automatic-namespace": {}
}
FILES = {
"src": ["src/example/__init__.py", "src/example/main.py"],
"flat": ["example/__init__.py", "example/main.py"],
"single_module": ["example.py"],
"namespace": ["ns/example/__init__.py"]
}

def _get_info(self, circumstance):
_, _, layout = circumstance.partition("-")
files = self.FILES[layout]
options = self.OPTIONS[circumstance]
metadata = self.METADATA
if layout == "namespace":
metadata = {**metadata, "name": "ns.example"}
return files, metadata, options

@pytest.mark.parametrize("circumstance", OPTIONS.keys())
def test_sdist_filelist(self, tmp_path, circumstance):
files, metadata, options = self._get_info(circumstance)
_populate_project_dir(tmp_path, files, metadata, options)

here = os.getcwd()
try:
os.chdir(tmp_path)
dist = Distribution({**metadata, **options, "src_root": tmp_path})
dist.script_name = 'setup.py'
dist.set_option_defaults()
cmd = sdist(dist)
cmd.ensure_finalized()
assert cmd.distribution.packages or cmd.distribution.py_modules

with quiet():
cmd.run()
finally:
os.chdir(here)

manifest = [f.replace(os.sep, "/") for f in cmd.filelist.files]
for file in files:
assert any(f.endswith(file) for f in manifest)

@pytest.mark.parametrize("circumstance", OPTIONS.keys())
def test_project(self, tmp_path, circumstance):
files, metadata, options = self._get_info(circumstance)
_populate_project_dir(tmp_path, files, metadata, options)

_run_build(tmp_path)

sdist_files = _get_sdist_members(next(tmp_path.glob("dist/*.tar.gz")))
print("~~~~~ sdist_members ~~~~~")
print('\n'.join(sdist_files))
assert sdist_files >= set(files)

wheel_files = _get_wheel_members(next(tmp_path.glob("dist/*.whl")))
print("~~~~~ wheel_members ~~~~~")
print('\n'.join(wheel_files))
assert wheel_files >= {f.replace("src/", "") for f in files}


def _populate_project_dir(root, files, metadata, options):
(root / "setup.py").write_text("import setuptools\nsetuptools.setup()")
(root / "README.md").write_text("# Example Package")
(root / "LICENSE").write_text("Copyright (c) 2018")
_write_setupcfg(root, metadata, options)

for file in files:
path = root / file
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text('"hello world"')


def _write_setupcfg(root, metadata, options):
setupcfg = ConfigParser()
setupcfg.add_section("metadata")
setupcfg["metadata"].update(metadata)
setupcfg.add_section("options")
for key, value in options.items():
if isinstance(value, list):
setupcfg["options"][key] = ", ".join(value)
elif isinstance(value, dict):
str_value = "\n".join(f"\t{k} = {v}" for k, v in value.items())
setupcfg["options"][key] = "\n" + str_value
else:
setupcfg["options"][key] = str(value)
with open(root / "setup.cfg", "w") as f:
setupcfg.write(f)
assert setupcfg["metadata"]["name"]
print("~~~~~ setup.cfg ~~~~~")
print((root / "setup.cfg").read_text())


def _get_sdist_members(sdist_path):
with tarfile.open(sdist_path, "r:gz") as tar:
files = [Path(f) for f in tar.getnames()]
relative_files = ("/".join(f.parts[1:]) for f in files)
# remove root folder
return {f for f in relative_files if f}


def _get_wheel_members(wheel_path):
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These 2 functions could be added to the Archive class defined in the test helper for PR #2863.

with ZipFile(wheel_path) as zipfile:
return set(zipfile.namelist())


def _run_build(path):
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the case PR #2863 gets merged, this function can be refactored in terms of the run function defined there.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That did get merged, it seems. :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @henryiii, thanks for the comment. I had forgotten about this one.

I ended up refactoring the code in #2894...

These 2 PRs have the same objective but different approaches.

Since #2894 goes in the direction pointed out in #2887, I think it has more chances of being merged in the future (I would like to have one of the two PRs merged once we add support for pyproject.toml metadata).

The improvements in #2894 can be easily ported over to this PR, if we think it is more suitable.

cmd = [sys.executable, "-m", "build", "--no-isolation", str(path)]
r = subprocess.run(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True,
env={**os.environ, 'DISTUTILS_DEBUG': '1'}
)
out = r.stdout + "\n" + r.stderr
print("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
print("Command", repr(cmd), "returncode", r.returncode)
print(out)

if r.returncode != 0:
raise CalledProcessError(r.returncode, cmd, r.stdout, r.stderr)
return out