diff --git a/setuptools/_core_metadata.py b/setuptools/_core_metadata.py new file mode 100644 index 0000000000..c1d41c6680 --- /dev/null +++ b/setuptools/_core_metadata.py @@ -0,0 +1,258 @@ +""" +Handling of Core Metadata for Python packages (including reading and writing). + +See: https://packaging.python.org/en/latest/specifications/core-metadata/ +""" +import os +import stat +import textwrap +from email import message_from_file +from email.message import Message +from tempfile import NamedTemporaryFile +from typing import Optional, List + +from distutils.util import rfc822_escape + +from . import _normalization +from .extern.packaging.markers import Marker +from .extern.packaging.requirements import Requirement +from .extern.packaging.version import Version +from .warnings import SetuptoolsDeprecationWarning + + +def get_metadata_version(self): + mv = getattr(self, 'metadata_version', None) + if mv is None: + mv = Version('2.1') + self.metadata_version = mv + return mv + + +def rfc822_unescape(content: str) -> str: + """Reverse RFC-822 escaping by removing leading whitespaces from content.""" + lines = content.splitlines() + if len(lines) == 1: + return lines[0].lstrip() + return '\n'.join((lines[0].lstrip(), textwrap.dedent('\n'.join(lines[1:])))) + + +def _read_field_from_msg(msg: Message, field: str) -> Optional[str]: + """Read Message header field.""" + value = msg[field] + if value == 'UNKNOWN': + return None + return value + + +def _read_field_unescaped_from_msg(msg: Message, field: str) -> Optional[str]: + """Read Message header field and apply rfc822_unescape.""" + value = _read_field_from_msg(msg, field) + if value is None: + return value + return rfc822_unescape(value) + + +def _read_list_from_msg(msg: Message, field: str) -> Optional[List[str]]: + """Read Message header field and return all results as list.""" + values = msg.get_all(field, None) + if values == []: + return None + return values + + +def _read_payload_from_msg(msg: Message) -> Optional[str]: + value = msg.get_payload().strip() + if value == 'UNKNOWN' or not value: + return None + return value + + +def read_pkg_file(self, file): + """Reads the metadata values from a file object.""" + msg = message_from_file(file) + + self.metadata_version = Version(msg['metadata-version']) + self.name = _read_field_from_msg(msg, 'name') + self.version = _read_field_from_msg(msg, 'version') + self.description = _read_field_from_msg(msg, 'summary') + # we are filling author only. + self.author = _read_field_from_msg(msg, 'author') + self.maintainer = None + self.author_email = _read_field_from_msg(msg, 'author-email') + self.maintainer_email = None + self.url = _read_field_from_msg(msg, 'home-page') + self.download_url = _read_field_from_msg(msg, 'download-url') + self.license = _read_field_unescaped_from_msg(msg, 'license') + + self.long_description = _read_field_unescaped_from_msg(msg, 'description') + if self.long_description is None and self.metadata_version >= Version('2.1'): + self.long_description = _read_payload_from_msg(msg) + self.description = _read_field_from_msg(msg, 'summary') + + if 'keywords' in msg: + self.keywords = _read_field_from_msg(msg, 'keywords').split(',') + + self.platforms = _read_list_from_msg(msg, 'platform') + self.classifiers = _read_list_from_msg(msg, 'classifier') + + # PEP 314 - these fields only exist in 1.1 + if self.metadata_version == Version('1.1'): + self.requires = _read_list_from_msg(msg, 'requires') + self.provides = _read_list_from_msg(msg, 'provides') + self.obsoletes = _read_list_from_msg(msg, 'obsoletes') + else: + self.requires = None + self.provides = None + self.obsoletes = None + + self.license_files = _read_list_from_msg(msg, 'license-file') + + +def single_line(val): + """ + Quick and dirty validation for Summary pypa/setuptools#1390. + """ + if '\n' in val: + # TODO: Replace with `raise ValueError("newlines not allowed")` + # after reviewing #2893. + msg = "newlines are not allowed in `summary` and will break in the future" + SetuptoolsDeprecationWarning.emit("Invalid config.", msg) + # due_date is undefined. Controversial change, there was a lot of push back. + val = val.strip().split('\n')[0] + return val + + +def write_pkg_info(self, base_dir): + """Write the PKG-INFO file into the release tree.""" + temp = "" + final = os.path.join(base_dir, 'PKG-INFO') + try: + # Use a temporary file while writing to avoid race conditions + # (e.g. `importlib.metadata` reading `.egg-info/PKG-INFO`): + with NamedTemporaryFile("w", encoding="utf-8", dir=base_dir, delete=False) as f: + temp = f.name + self.write_pkg_file(f) + permissions = stat.S_IMODE(os.lstat(temp).st_mode) + os.chmod(temp, permissions | stat.S_IRGRP | stat.S_IROTH) + os.replace(temp, final) # atomic operation. + finally: + if temp and os.path.exists(temp): + os.remove(temp) + + +# Based on Python 3.5 version +def write_pkg_file(self, file): # noqa: C901 # is too complex (14) # FIXME + """Write the PKG-INFO format data to a file object.""" + version = self.get_metadata_version() + + def write_field(key, value): + file.write("%s: %s\n" % (key, value)) + + write_field('Metadata-Version', str(version)) + write_field('Name', self.get_name()) + write_field('Version', self.get_version()) + + summary = self.get_description() + if summary: + write_field('Summary', single_line(summary)) + + optional_fields = ( + ('Home-page', 'url'), + ('Download-URL', 'download_url'), + ('Author', 'author'), + ('Author-email', 'author_email'), + ('Maintainer', 'maintainer'), + ('Maintainer-email', 'maintainer_email'), + ) + + for field, attr in optional_fields: + attr_val = getattr(self, attr, None) + if attr_val is not None: + write_field(field, attr_val) + + license = self.get_license() + if license: + write_field('License', rfc822_escape(license)) + + for project_url in self.project_urls.items(): + write_field('Project-URL', '%s, %s' % project_url) + + keywords = ','.join(self.get_keywords()) + if keywords: + write_field('Keywords', keywords) + + platforms = self.get_platforms() or [] + for platform in platforms: + write_field('Platform', platform) + + self._write_list(file, 'Classifier', self.get_classifiers()) + + # PEP 314 + self._write_list(file, 'Requires', self.get_requires()) + self._write_list(file, 'Provides', self.get_provides()) + self._write_list(file, 'Obsoletes', self.get_obsoletes()) + + # Setuptools specific for PEP 345 + if hasattr(self, 'python_requires'): + write_field('Requires-Python', self.python_requires) + + # PEP 566 + if self.long_description_content_type: + write_field('Description-Content-Type', self.long_description_content_type) + + self._write_list(file, 'License-File', self.license_files or []) + _write_requirements(self, file) + + long_description = self.get_long_description() + if long_description: + file.write("\n%s" % long_description) + if not long_description.endswith("\n"): + file.write("\n") + + +def _write_requirements(self, file): + for req in self._normalized_install_requires: + file.write(f"Requires-Dist: {req}\n") + + processed_extras = {} + for augmented_extra, reqs in self._normalized_extras_require.items(): + # Historically, setuptools allows "augmented extras": `:` + unsafe_extra, _, condition = augmented_extra.partition(":") + unsafe_extra = unsafe_extra.strip() + extra = _normalization.safe_extra(unsafe_extra) + + if extra: + _write_provides_extra(file, processed_extras, extra, unsafe_extra) + for req in reqs: + r = _include_extra(req, extra, condition.strip()) + file.write(f"Requires-Dist: {r}\n") + + return processed_extras + + +def _include_extra(req: str, extra: str, condition: str) -> Requirement: + r = Requirement(req) + parts = ( + f"({r.marker})" if r.marker else None, + f"({condition})" if condition else None, + f"extra == {extra!r}" if extra else None, + ) + r.marker = Marker(" and ".join(x for x in parts if x)) + return r + + +def _write_provides_extra(file, processed_extras, safe, unsafe): + previous = processed_extras.get(safe) + if previous == unsafe: + SetuptoolsDeprecationWarning.emit( + 'Ambiguity during "extra" normalization for dependencies.', + f""" + {previous!r} and {unsafe!r} normalize to the same value:\n + {safe!r}\n + In future versions, setuptools might halt the build process. + """, + see_url="https://peps.python.org/pep-0685/", + ) + else: + processed_extras[safe] = unsafe + file.write(f"Provides-Extra: {safe}\n") diff --git a/setuptools/_normalization.py b/setuptools/_normalization.py index 31899f7ab1..3e94e662ef 100644 --- a/setuptools/_normalization.py +++ b/setuptools/_normalization.py @@ -14,6 +14,7 @@ # https://packaging.python.org/en/latest/specifications/core-metadata/#name _VALID_NAME = re.compile(r"^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$", re.I) _UNSAFE_NAME_CHARS = re.compile(r"[^A-Z0-9.]+", re.I) +_NON_ALPHANUMERIC = re.compile(r"[^A-Z0-9]+", re.I) def safe_identifier(name: str) -> str: @@ -92,6 +93,16 @@ def best_effort_version(version: str) -> str: return safe_name(v) +def safe_extra(extra: str) -> str: + """Normalize extra name according to PEP 685 + >>> safe_extra("_FrIeNdLy-._.-bArD") + 'friendly-bard' + >>> safe_extra("FrIeNdLy-._.-bArD__._-") + 'friendly-bard' + """ + return _NON_ALPHANUMERIC.sub("-", extra).strip("-").lower() + + def filename_component(value: str) -> str: """Normalize each component of a filename (e.g. distribution/version part of wheel) Note: ``value`` needs to be already normalized. diff --git a/setuptools/command/_requirestxt.py b/setuptools/command/_requirestxt.py index 435d71dca5..d223737fd4 100644 --- a/setuptools/command/_requirestxt.py +++ b/setuptools/command/_requirestxt.py @@ -11,7 +11,6 @@ from itertools import filterfalse from typing import Dict, List, Tuple, Mapping, TypeVar -from .. import _reqs from ..extern.jaraco.text import yield_lines from ..extern.packaging.requirements import Requirement @@ -20,11 +19,11 @@ _T = TypeVar("_T") _Ordered = Dict[_T, None] _ordered = dict -_StrOrIter = _reqs._StrOrIter def _prepare( - install_requires: _StrOrIter, extras_require: Mapping[str, _StrOrIter] + install_requires: Dict[str, Requirement], + extras_require: Mapping[str, Dict[str, Requirement]], ) -> Tuple[List[str], Dict[str, List[str]]]: """Given values for ``install_requires`` and ``extras_require`` create modified versions in a way that can be written in ``requires.txt`` @@ -34,7 +33,7 @@ def _prepare( def _convert_extras_requirements( - extras_require: _StrOrIter, + extras_require: Dict[str, Dict[str, Requirement]], ) -> Mapping[str, _Ordered[Requirement]]: """ Convert requirements in `extras_require` of the form @@ -45,14 +44,15 @@ def _convert_extras_requirements( for section, v in extras_require.items(): # Do not strip empty sections. output[section] - for r in _reqs.parse(v): + for r in v.values(): output[section + _suffix_for(r)].setdefault(r) return output def _move_install_requirements_markers( - install_requires: _StrOrIter, extras_require: Mapping[str, _Ordered[Requirement]] + install_requires: Dict[str, Requirement], + extras_require: Mapping[str, _Ordered[Requirement]], ) -> Tuple[List[str], Dict[str, List[str]]]: """ The ``requires.txt`` file has an specific format: @@ -66,7 +66,7 @@ def _move_install_requirements_markers( # divide the install_requires into two sets, simple ones still # handled by install_requires and more complex ones handled by extras_require. - inst_reqs = list(_reqs.parse(install_requires)) + inst_reqs = install_requires.values() simple_reqs = filter(_no_marker, inst_reqs) complex_reqs = filterfalse(_no_marker, inst_reqs) simple_install_requires = list(map(str, simple_reqs)) @@ -90,8 +90,9 @@ def _suffix_for(req): def _clean_req(req): """Given a Requirement, remove environment markers and return it""" - req.marker = None - return req + r = Requirement(str(req)) # create a copy before modifying. + r.marker = None + return r def _no_marker(req): @@ -110,9 +111,10 @@ def append_cr(line): def write_requirements(cmd, basename, filename): dist = cmd.distribution + meta = dist.metadata data = io.StringIO() install_requires, extras_require = _prepare( - dist.install_requires or (), dist.extras_require or {} + meta._normalized_install_requires, meta._normalized_extras_require ) _write_requirements(data, install_requires) for extra in sorted(extras_require): diff --git a/setuptools/dist.py b/setuptools/dist.py index 5e05920356..f1d361f1c3 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -7,12 +7,10 @@ import os import re import sys -import textwrap from contextlib import suppress -from email import message_from_file from glob import iglob from pathlib import Path -from typing import TYPE_CHECKING, List, Optional, Set +from typing import List, Optional, Set import distutils.cmd import distutils.command @@ -22,7 +20,6 @@ from distutils.debug import DEBUG from distutils.errors import DistutilsOptionError, DistutilsSetupError from distutils.fancy_getopt import translate_longopt -from distutils.util import rfc822_escape from distutils.util import strtobool from .extern.more_itertools import partition, unique_everseen @@ -42,184 +39,6 @@ from .warnings import InformationOnly, SetuptoolsDeprecationWarning -if TYPE_CHECKING: - from email.message import Message - - -def get_metadata_version(self): - mv = getattr(self, 'metadata_version', None) - if mv is None: - mv = Version('2.1') - self.metadata_version = mv - return mv - - -def rfc822_unescape(content: str) -> str: - """Reverse RFC-822 escaping by removing leading whitespaces from content.""" - lines = content.splitlines() - if len(lines) == 1: - return lines[0].lstrip() - return '\n'.join((lines[0].lstrip(), textwrap.dedent('\n'.join(lines[1:])))) - - -def _read_field_from_msg(msg: "Message", field: str) -> Optional[str]: - """Read Message header field.""" - value = msg[field] - if value == 'UNKNOWN': - return None - return value - - -def _read_field_unescaped_from_msg(msg: "Message", field: str) -> Optional[str]: - """Read Message header field and apply rfc822_unescape.""" - value = _read_field_from_msg(msg, field) - if value is None: - return value - return rfc822_unescape(value) - - -def _read_list_from_msg(msg: "Message", field: str) -> Optional[List[str]]: - """Read Message header field and return all results as list.""" - values = msg.get_all(field, None) - if values == []: - return None - return values - - -def _read_payload_from_msg(msg: "Message") -> Optional[str]: - value = msg.get_payload().strip() - if value == 'UNKNOWN' or not value: - return None - return value - - -def read_pkg_file(self, file): - """Reads the metadata values from a file object.""" - msg = message_from_file(file) - - self.metadata_version = Version(msg['metadata-version']) - self.name = _read_field_from_msg(msg, 'name') - self.version = _read_field_from_msg(msg, 'version') - self.description = _read_field_from_msg(msg, 'summary') - # we are filling author only. - self.author = _read_field_from_msg(msg, 'author') - self.maintainer = None - self.author_email = _read_field_from_msg(msg, 'author-email') - self.maintainer_email = None - self.url = _read_field_from_msg(msg, 'home-page') - self.download_url = _read_field_from_msg(msg, 'download-url') - self.license = _read_field_unescaped_from_msg(msg, 'license') - - self.long_description = _read_field_unescaped_from_msg(msg, 'description') - if self.long_description is None and self.metadata_version >= Version('2.1'): - self.long_description = _read_payload_from_msg(msg) - self.description = _read_field_from_msg(msg, 'summary') - - if 'keywords' in msg: - self.keywords = _read_field_from_msg(msg, 'keywords').split(',') - - self.platforms = _read_list_from_msg(msg, 'platform') - self.classifiers = _read_list_from_msg(msg, 'classifier') - - # PEP 314 - these fields only exist in 1.1 - if self.metadata_version == Version('1.1'): - self.requires = _read_list_from_msg(msg, 'requires') - self.provides = _read_list_from_msg(msg, 'provides') - self.obsoletes = _read_list_from_msg(msg, 'obsoletes') - else: - self.requires = None - self.provides = None - self.obsoletes = None - - self.license_files = _read_list_from_msg(msg, 'license-file') - - -def single_line(val): - """ - Quick and dirty validation for Summary pypa/setuptools#1390. - """ - if '\n' in val: - # TODO: Replace with `raise ValueError("newlines not allowed")` - # after reviewing #2893. - msg = "newlines are not allowed in `summary` and will break in the future" - SetuptoolsDeprecationWarning.emit("Invalid config.", msg) - # due_date is undefined. Controversial change, there was a lot of push back. - val = val.strip().split('\n')[0] - return val - - -# Based on Python 3.5 version -def write_pkg_file(self, file): # noqa: C901 # is too complex (14) # FIXME - """Write the PKG-INFO format data to a file object.""" - version = self.get_metadata_version() - - def write_field(key, value): - file.write("%s: %s\n" % (key, value)) - - write_field('Metadata-Version', str(version)) - write_field('Name', self.get_name()) - write_field('Version', self.get_version()) - - summary = self.get_description() - if summary: - write_field('Summary', single_line(summary)) - - optional_fields = ( - ('Home-page', 'url'), - ('Download-URL', 'download_url'), - ('Author', 'author'), - ('Author-email', 'author_email'), - ('Maintainer', 'maintainer'), - ('Maintainer-email', 'maintainer_email'), - ) - - for field, attr in optional_fields: - attr_val = getattr(self, attr, None) - if attr_val is not None: - write_field(field, attr_val) - - license = self.get_license() - if license: - write_field('License', rfc822_escape(license)) - - for project_url in self.project_urls.items(): - write_field('Project-URL', '%s, %s' % project_url) - - keywords = ','.join(self.get_keywords()) - if keywords: - write_field('Keywords', keywords) - - platforms = self.get_platforms() or [] - for platform in platforms: - write_field('Platform', platform) - - self._write_list(file, 'Classifier', self.get_classifiers()) - - # PEP 314 - self._write_list(file, 'Requires', self.get_requires()) - self._write_list(file, 'Provides', self.get_provides()) - self._write_list(file, 'Obsoletes', self.get_obsoletes()) - - # Setuptools specific for PEP 345 - if hasattr(self, 'python_requires'): - write_field('Requires-Python', self.python_requires) - - # PEP 566 - if self.long_description_content_type: - write_field('Description-Content-Type', self.long_description_content_type) - if self.provides_extras: - for extra in self.provides_extras: - write_field('Provides-Extra', extra) - - self._write_list(file, 'License-File', self.license_files or []) - - long_description = self.get_long_description() - if long_description: - file.write("\n%s" % long_description) - if not long_description.endswith("\n"): - file.write("\n") - - sequence = tuple, list @@ -444,6 +263,11 @@ class Distribution(_Distribution): 'provides_extras': OrderedSet, 'license_file': lambda: None, 'license_files': lambda: None, + # Both install_requires and extras_require are needed to write PKG-INFO, + # So we take this opportunity to cache parsed requirement objects. + # These attributes are not part of the public API and intended for internal use. + '_normalized_install_requires': dict, # Dict[str, Requirement] + '_normalized_extras_require': dict, # Dict[str, Dict[str, Requirement]] } _patched_dist = None @@ -582,9 +406,17 @@ def _normalize_requires(self): """Make sure requirement-related attributes exist and are normalized""" install_requires = getattr(self, "install_requires", None) or [] extras_require = getattr(self, "extras_require", None) or {} - self.install_requires = list(map(str, _reqs.parse(install_requires))) + meta = self.metadata + meta._normalized_install_requires = { + str(r): r for r in _reqs.parse(install_requires) + } + meta._normalized_extras_require = { + k: {str(r): r for r in _reqs.parse(v or [])} + for k, v in extras_require.items() + } + self.install_requires = list(meta._normalized_install_requires) self.extras_require = { - k: list(map(str, _reqs.parse(v or []))) for k, v in extras_require.items() + k: list(v) for k, v in meta._normalized_extras_require.items() } def _finalize_license_files(self): diff --git a/setuptools/monkey.py b/setuptools/monkey.py index 9464de46ab..2ab98c178a 100644 --- a/setuptools/monkey.py +++ b/setuptools/monkey.py @@ -99,11 +99,16 @@ def patch_all(): def _patch_distribution_metadata(): - from . import dist + from . import _core_metadata """Patch write_pkg_file and read_pkg_file for higher metadata standards""" - for attr in ('write_pkg_file', 'read_pkg_file', 'get_metadata_version'): - new_val = getattr(dist, attr) + for attr in ( + 'write_pkg_info', + 'write_pkg_file', + 'read_pkg_file', + 'get_metadata_version', + ): + new_val = getattr(_core_metadata, attr) setattr(distutils.dist.DistributionMetadata, attr, new_val) diff --git a/setuptools/tests/config/test_apply_pyprojecttoml.py b/setuptools/tests/config/test_apply_pyprojecttoml.py index 8a654ff94a..50099df491 100644 --- a/setuptools/tests/config/test_apply_pyprojecttoml.py +++ b/setuptools/tests/config/test_apply_pyprojecttoml.py @@ -45,6 +45,7 @@ def test_apply_pyproject_equivalent_to_setupcfg(url, monkeypatch, tmp_path): dist_toml = pyprojecttoml.apply_configuration(makedist(tmp_path), pyproject_example) dist_cfg = setupcfg.apply_configuration(makedist(tmp_path), setupcfg_example) + _port_tests_require(dist_cfg) pkg_info_toml = core_metadata(dist_toml) pkg_info_cfg = core_metadata(dist_cfg) @@ -448,3 +449,15 @@ def core_metadata(dist) -> str: result.append(line + "\n") return "".join(result) + + +def _port_tests_require(dist): + """ + ``ini2toml`` "forward fix" deprecated tests_require definitions by moving + them into an extra called ``testing``. + """ + tests_require = getattr(dist, "tests_require", None) or [] + if tests_require: + dist.tests_require = [] + dist.extras_require.setdefault("testing", []).extend(tests_require) + dist._finalize_requires() diff --git a/setuptools/tests/test_core_metadata.py b/setuptools/tests/test_core_metadata.py new file mode 100644 index 0000000000..5c7982a71b --- /dev/null +++ b/setuptools/tests/test_core_metadata.py @@ -0,0 +1,380 @@ +import functools +import io +import importlib +from email import message_from_string + +import pytest + +from setuptools import sic, _reqs +from setuptools.dist import Distribution +from setuptools._core_metadata import rfc822_escape, rfc822_unescape +from setuptools.command.egg_info import egg_info, write_requirements + + +EXAMPLE_BASE_INFO = dict( + name="package", + version="0.0.1", + author="Foo Bar", + author_email="foo@bar.net", + long_description="Long\ndescription", + description="Short description", + keywords=["one", "two"], +) + + +@pytest.mark.parametrize( + 'content, result', + ( + pytest.param( + "Just a single line", + None, + id="single_line", + ), + pytest.param( + "Multiline\nText\nwithout\nextra indents\n", + None, + id="multiline", + ), + pytest.param( + "Multiline\n With\n\nadditional\n indentation", + None, + id="multiline_with_indentation", + ), + pytest.param( + " Leading whitespace", + "Leading whitespace", + id="remove_leading_whitespace", + ), + pytest.param( + " Leading whitespace\nIn\n Multiline comment", + "Leading whitespace\nIn\n Multiline comment", + id="remove_leading_whitespace_multiline", + ), + ), +) +def test_rfc822_unescape(content, result): + assert (result or content) == rfc822_unescape(rfc822_escape(content)) + + +def __read_test_cases(): + base = EXAMPLE_BASE_INFO + + params = functools.partial(dict, base) + + test_cases = [ + ('Metadata version 1.0', params()), + ( + 'Metadata Version 1.0: Short long description', + params( + long_description='Short long description', + ), + ), + ( + 'Metadata version 1.1: Classifiers', + params( + classifiers=[ + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'License :: OSI Approved :: MIT License', + ], + ), + ), + ( + 'Metadata version 1.1: Download URL', + params( + download_url='https://example.com', + ), + ), + ( + 'Metadata Version 1.2: Requires-Python', + params( + python_requires='>=3.7', + ), + ), + pytest.param( + 'Metadata Version 1.2: Project-Url', + params(project_urls=dict(Foo='https://example.bar')), + marks=pytest.mark.xfail( + reason="Issue #1578: project_urls not read", + ), + ), + ( + 'Metadata Version 2.1: Long Description Content Type', + params( + long_description_content_type='text/x-rst; charset=UTF-8', + ), + ), + ( + 'License', + params( + license='MIT', + ), + ), + ( + 'License multiline', + params( + license='This is a long license \nover multiple lines', + ), + ), + pytest.param( + 'Metadata Version 2.1: Provides Extra', + params(provides_extras=['foo', 'bar']), + marks=pytest.mark.xfail(reason="provides_extras not read"), + ), + ( + 'Missing author', + dict( + name='foo', + version='1.0.0', + author_email='snorri@sturluson.name', + ), + ), + ( + 'Missing author e-mail', + dict( + name='foo', + version='1.0.0', + author='Snorri Sturluson', + ), + ), + ( + 'Missing author and e-mail', + dict( + name='foo', + version='1.0.0', + ), + ), + ( + 'Bypass normalized version', + dict( + name='foo', + version=sic('1.0.0a'), + ), + ), + ] + + return test_cases + + +@pytest.mark.parametrize('name,attrs', __read_test_cases()) +def test_read_metadata(name, attrs): + dist = Distribution(attrs) + metadata_out = dist.metadata + dist_class = metadata_out.__class__ + + # Write to PKG_INFO and then load into a new metadata object + PKG_INFO = io.StringIO() + + metadata_out.write_pkg_file(PKG_INFO) + + PKG_INFO.seek(0) + metadata_in = dist_class() + metadata_in.read_pkg_file(PKG_INFO) + + tested_attrs = [ + ('name', dist_class.get_name), + ('version', dist_class.get_version), + ('author', dist_class.get_contact), + ('author_email', dist_class.get_contact_email), + ('metadata_version', dist_class.get_metadata_version), + ('provides', dist_class.get_provides), + ('description', dist_class.get_description), + ('long_description', dist_class.get_long_description), + ('download_url', dist_class.get_download_url), + ('keywords', dist_class.get_keywords), + ('platforms', dist_class.get_platforms), + ('obsoletes', dist_class.get_obsoletes), + ('requires', dist_class.get_requires), + ('classifiers', dist_class.get_classifiers), + ('project_urls', lambda s: getattr(s, 'project_urls', {})), + ('provides_extras', lambda s: getattr(s, 'provides_extras', set())), + ] + + for attr, getter in tested_attrs: + assert getter(metadata_in) == getter(metadata_out) + + +def __maintainer_test_cases(): + attrs = {"name": "package", "version": "1.0", "description": "xxx"} + + def merge_dicts(d1, d2): + d1 = d1.copy() + d1.update(d2) + + return d1 + + test_cases = [ + ('No author, no maintainer', attrs.copy()), + ( + 'Author (no e-mail), no maintainer', + merge_dicts(attrs, {'author': 'Author Name'}), + ), + ( + 'Author (e-mail), no maintainer', + merge_dicts( + attrs, {'author': 'Author Name', 'author_email': 'author@name.com'} + ), + ), + ( + 'No author, maintainer (no e-mail)', + merge_dicts(attrs, {'maintainer': 'Maintainer Name'}), + ), + ( + 'No author, maintainer (e-mail)', + merge_dicts( + attrs, + { + 'maintainer': 'Maintainer Name', + 'maintainer_email': 'maintainer@name.com', + }, + ), + ), + ( + 'Author (no e-mail), Maintainer (no-email)', + merge_dicts( + attrs, {'author': 'Author Name', 'maintainer': 'Maintainer Name'} + ), + ), + ( + 'Author (e-mail), Maintainer (e-mail)', + merge_dicts( + attrs, + { + 'author': 'Author Name', + 'author_email': 'author@name.com', + 'maintainer': 'Maintainer Name', + 'maintainer_email': 'maintainer@name.com', + }, + ), + ), + ( + 'No author (e-mail), no maintainer (e-mail)', + merge_dicts( + attrs, + { + 'author_email': 'author@name.com', + 'maintainer_email': 'maintainer@name.com', + }, + ), + ), + ('Author unicode', merge_dicts(attrs, {'author': '鉄沢寛'})), + ('Maintainer unicode', merge_dicts(attrs, {'maintainer': 'Jan Łukasiewicz'})), + ] + + return test_cases + + +@pytest.mark.parametrize('name,attrs', __maintainer_test_cases()) +def test_maintainer_author(name, attrs, tmpdir): + tested_keys = { + 'author': 'Author', + 'author_email': 'Author-email', + 'maintainer': 'Maintainer', + 'maintainer_email': 'Maintainer-email', + } + + # Generate a PKG-INFO file + dist = Distribution(attrs) + fn = tmpdir.mkdir('pkg_info') + fn_s = str(fn) + + dist.metadata.write_pkg_info(fn_s) + + with io.open(str(fn.join('PKG-INFO')), 'r', encoding='utf-8') as f: + raw_pkg_lines = f.readlines() + + # Drop blank lines and strip lines from default description + pkg_lines = list(filter(None, raw_pkg_lines[:-2])) + + pkg_lines_set = set(pkg_lines) + + # Duplicate lines should not be generated + assert len(pkg_lines) == len(pkg_lines_set) + + for fkey, dkey in tested_keys.items(): + val = attrs.get(dkey, None) + if val is None: + for line in pkg_lines: + assert not line.startswith(fkey + ':') + else: + line = '%s: %s' % (fkey, val) + assert line in pkg_lines_set + + +def test_parity_with_metadata_from_pypa_wheel(tmp_path): + attrs = dict( + **EXAMPLE_BASE_INFO, + # Example with complex requirement definition + python_requires=">=3.8", + install_requires=""" + packaging==23.0 + ordered-set==3.1.1 + more-itertools==8.8.0; extra == "other" + jaraco.text==3.7.0 + importlib-resources==5.10.2; python_version<"3.8" + importlib-metadata==6.0.0 ; python_version<"3.8" + colorama>=0.4.4; sys_platform == "win32" + """, + extras_require={ + "testing": """ + pytest >= 6 + pytest-checkdocs >= 2.4 + pytest-flake8 ; \\ + # workaround for tholo/pytest-flake8#87 + python_version < "3.12" + ini2toml[lite]>=0.9 + """, + "other": [], + } + ) + # Generate a PKG-INFO file using setuptools + dist = Distribution(attrs) + with io.StringIO() as fp: + dist.metadata.write_pkg_file(fp) + pkg_info = fp.getvalue() + + # Ensure Requires-Dist is present + expected = [ + 'Metadata-Version:', + 'Requires-Python: >=3.8', + 'Provides-Extra: other', + 'Provides-Extra: testing', + 'Requires-Dist: pytest-flake8; python_version < "3.12" and extra == "testing"', + 'Requires-Dist: more-itertools==8.8.0; extra == "other"', + 'Requires-Dist: ini2toml[lite]>=0.9; extra == "testing"', + ] + for line in expected: + assert line in pkg_info + + # Generate a METADATA file using pypa/wheel for comparisson + wheel_metadata = importlib.import_module("wheel.metadata") + pkginfo_to_metadata = getattr(wheel_metadata, "pkginfo_to_metadata", None) + + if pkginfo_to_metadata is None: + pytest.xfail( + "wheel.metadata.pkginfo_to_metadata is undefined, " + "(this is likely to be caused by API changes in pypa/wheel" + ) + + # Generate an simplified "egg-info" dir for pypa/wheel to convert + egg_info_dir = tmp_path / "pkg.egg-info" + egg_info_dir.mkdir(parents=True) + (egg_info_dir / "PKG-INFO").write_text(pkg_info, encoding="utf-8") + write_requirements(egg_info(dist), egg_info_dir, egg_info_dir / "requires.txt") + + # Get pypa/wheel generated METADATA but normalize requirements formatting + metadata_msg = pkginfo_to_metadata(egg_info_dir, egg_info_dir / "PKG-INFO") + metadata_deps = set(_reqs.parse(metadata_msg.get_all("Requires-Dist"))) + metadata_extras = set(metadata_msg.get_all("Provides-Extra")) + del metadata_msg["Requires-Dist"] + del metadata_msg["Provides-Extra"] + pkg_info_msg = message_from_string(pkg_info) + pkg_info_deps = set(_reqs.parse(pkg_info_msg.get_all("Requires-Dist"))) + pkg_info_extras = set(pkg_info_msg.get_all("Provides-Extra")) + del pkg_info_msg["Requires-Dist"] + del pkg_info_msg["Provides-Extra"] + + # Compare setuptools PKG-INFO x pypa/wheel METADATA + assert metadata_msg.as_string() == pkg_info_msg.as_string() + assert metadata_deps == pkg_info_deps + assert metadata_extras == pkg_info_extras diff --git a/setuptools/tests/test_dist.py b/setuptools/tests/test_dist.py index f8f7996486..0caaef6578 100644 --- a/setuptools/tests/test_dist.py +++ b/setuptools/tests/test_dist.py @@ -1,7 +1,5 @@ -import io import collections import re -import functools import os import urllib.request import urllib.parse @@ -9,10 +7,7 @@ from setuptools.dist import ( check_package_data, check_specifier, - rfc822_escape, - rfc822_unescape, ) -from setuptools import sic from setuptools import Distribution from .textwrap import DALS @@ -77,251 +72,6 @@ def sdist_with_index(distname, version): ) -def __read_test_cases(): - base = EXAMPLE_BASE_INFO - - params = functools.partial(dict, base) - - test_cases = [ - ('Metadata version 1.0', params()), - ( - 'Metadata Version 1.0: Short long description', - params( - long_description='Short long description', - ), - ), - ( - 'Metadata version 1.1: Classifiers', - params( - classifiers=[ - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', - 'License :: OSI Approved :: MIT License', - ], - ), - ), - ( - 'Metadata version 1.1: Download URL', - params( - download_url='https://example.com', - ), - ), - ( - 'Metadata Version 1.2: Requires-Python', - params( - python_requires='>=3.7', - ), - ), - pytest.param( - 'Metadata Version 1.2: Project-Url', - params(project_urls=dict(Foo='https://example.bar')), - marks=pytest.mark.xfail( - reason="Issue #1578: project_urls not read", - ), - ), - ( - 'Metadata Version 2.1: Long Description Content Type', - params( - long_description_content_type='text/x-rst; charset=UTF-8', - ), - ), - ( - 'License', - params( - license='MIT', - ), - ), - ( - 'License multiline', - params( - license='This is a long license \nover multiple lines', - ), - ), - pytest.param( - 'Metadata Version 2.1: Provides Extra', - params(provides_extras=['foo', 'bar']), - marks=pytest.mark.xfail(reason="provides_extras not read"), - ), - ( - 'Missing author', - dict( - name='foo', - version='1.0.0', - author_email='snorri@sturluson.name', - ), - ), - ( - 'Missing author e-mail', - dict( - name='foo', - version='1.0.0', - author='Snorri Sturluson', - ), - ), - ( - 'Missing author and e-mail', - dict( - name='foo', - version='1.0.0', - ), - ), - ( - 'Bypass normalized version', - dict( - name='foo', - version=sic('1.0.0a'), - ), - ), - ] - - return test_cases - - -@pytest.mark.parametrize('name,attrs', __read_test_cases()) -def test_read_metadata(name, attrs): - dist = Distribution(attrs) - metadata_out = dist.metadata - dist_class = metadata_out.__class__ - - # Write to PKG_INFO and then load into a new metadata object - PKG_INFO = io.StringIO() - - metadata_out.write_pkg_file(PKG_INFO) - - PKG_INFO.seek(0) - metadata_in = dist_class() - metadata_in.read_pkg_file(PKG_INFO) - - tested_attrs = [ - ('name', dist_class.get_name), - ('version', dist_class.get_version), - ('author', dist_class.get_contact), - ('author_email', dist_class.get_contact_email), - ('metadata_version', dist_class.get_metadata_version), - ('provides', dist_class.get_provides), - ('description', dist_class.get_description), - ('long_description', dist_class.get_long_description), - ('download_url', dist_class.get_download_url), - ('keywords', dist_class.get_keywords), - ('platforms', dist_class.get_platforms), - ('obsoletes', dist_class.get_obsoletes), - ('requires', dist_class.get_requires), - ('classifiers', dist_class.get_classifiers), - ('project_urls', lambda s: getattr(s, 'project_urls', {})), - ('provides_extras', lambda s: getattr(s, 'provides_extras', set())), - ] - - for attr, getter in tested_attrs: - assert getter(metadata_in) == getter(metadata_out) - - -def __maintainer_test_cases(): - attrs = {"name": "package", "version": "1.0", "description": "xxx"} - - def merge_dicts(d1, d2): - d1 = d1.copy() - d1.update(d2) - - return d1 - - test_cases = [ - ('No author, no maintainer', attrs.copy()), - ( - 'Author (no e-mail), no maintainer', - merge_dicts(attrs, {'author': 'Author Name'}), - ), - ( - 'Author (e-mail), no maintainer', - merge_dicts( - attrs, {'author': 'Author Name', 'author_email': 'author@name.com'} - ), - ), - ( - 'No author, maintainer (no e-mail)', - merge_dicts(attrs, {'maintainer': 'Maintainer Name'}), - ), - ( - 'No author, maintainer (e-mail)', - merge_dicts( - attrs, - { - 'maintainer': 'Maintainer Name', - 'maintainer_email': 'maintainer@name.com', - }, - ), - ), - ( - 'Author (no e-mail), Maintainer (no-email)', - merge_dicts( - attrs, {'author': 'Author Name', 'maintainer': 'Maintainer Name'} - ), - ), - ( - 'Author (e-mail), Maintainer (e-mail)', - merge_dicts( - attrs, - { - 'author': 'Author Name', - 'author_email': 'author@name.com', - 'maintainer': 'Maintainer Name', - 'maintainer_email': 'maintainer@name.com', - }, - ), - ), - ( - 'No author (e-mail), no maintainer (e-mail)', - merge_dicts( - attrs, - { - 'author_email': 'author@name.com', - 'maintainer_email': 'maintainer@name.com', - }, - ), - ), - ('Author unicode', merge_dicts(attrs, {'author': '鉄沢寛'})), - ('Maintainer unicode', merge_dicts(attrs, {'maintainer': 'Jan Łukasiewicz'})), - ] - - return test_cases - - -@pytest.mark.parametrize('name,attrs', __maintainer_test_cases()) -def test_maintainer_author(name, attrs, tmpdir): - tested_keys = { - 'author': 'Author', - 'author_email': 'Author-email', - 'maintainer': 'Maintainer', - 'maintainer_email': 'Maintainer-email', - } - - # Generate a PKG-INFO file - dist = Distribution(attrs) - fn = tmpdir.mkdir('pkg_info') - fn_s = str(fn) - - dist.metadata.write_pkg_info(fn_s) - - with io.open(str(fn.join('PKG-INFO')), 'r', encoding='utf-8') as f: - raw_pkg_lines = f.readlines() - - # Drop blank lines and strip lines from default description - pkg_lines = list(filter(None, raw_pkg_lines[:-2])) - - pkg_lines_set = set(pkg_lines) - - # Duplicate lines should not be generated - assert len(pkg_lines) == len(pkg_lines_set) - - for fkey, dkey in tested_keys.items(): - val = attrs.get(dkey, None) - if val is None: - for line in pkg_lines: - assert not line.startswith(fkey + ':') - else: - line = '%s: %s' % (fkey, val) - assert line in pkg_lines_set - - def test_provides_extras_deterministic_order(): extras = collections.OrderedDict() extras['a'] = ['foo'] @@ -407,40 +157,6 @@ def test_check_specifier(): dist = Distribution(attrs) -@pytest.mark.parametrize( - 'content, result', - ( - pytest.param( - "Just a single line", - None, - id="single_line", - ), - pytest.param( - "Multiline\nText\nwithout\nextra indents\n", - None, - id="multiline", - ), - pytest.param( - "Multiline\n With\n\nadditional\n indentation", - None, - id="multiline_with_indentation", - ), - pytest.param( - " Leading whitespace", - "Leading whitespace", - id="remove_leading_whitespace", - ), - pytest.param( - " Leading whitespace\nIn\n Multiline comment", - "Leading whitespace\nIn\n Multiline comment", - id="remove_leading_whitespace_multiline", - ), - ), -) -def test_rfc822_unescape(content, result): - assert (result or content) == rfc822_unescape(rfc822_escape(content)) - - def test_metadata_name(): with pytest.raises(DistutilsSetupError, match='missing.*name'): Distribution()._validate_metadata()