From aa616bf7f94d1f398d365ac488f22fb501e124f6 Mon Sep 17 00:00:00 2001 From: James Hilliard Date: Thu, 20 Jan 2022 20:44:34 -0700 Subject: [PATCH] flit_core: refactor path handling to use pathlib --- flit_core/flit_core/buildapi.py | 23 ++++----- flit_core/flit_core/common.py | 38 +++++++-------- flit_core/flit_core/config.py | 46 ++++++++++++++---- flit_core/flit_core/sdist.py | 62 +++++++++++-------------- flit_core/flit_core/tests/test_sdist.py | 7 ++- flit_core/flit_core/wheel.py | 33 ++++++------- 6 files changed, 108 insertions(+), 101 deletions(-) diff --git a/flit_core/flit_core/buildapi.py b/flit_core/flit_core/buildapi.py index 963bf614..fc92cc98 100644 --- a/flit_core/flit_core/buildapi.py +++ b/flit_core/flit_core/buildapi.py @@ -1,8 +1,5 @@ """PEP-517 compliant buildsystem API""" import logging -import io -import os -import os.path as osp from pathlib import Path from .common import ( @@ -48,36 +45,36 @@ def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): module = Module(ini_info.module, Path.cwd()) metadata = make_metadata(module, ini_info) - dist_info = osp.join(metadata_directory, - dist_info_name(metadata.name, metadata.version)) - os.mkdir(dist_info) + dist_info = Path(metadata_directory).joinpath( + dist_info_name(metadata.name, metadata.version)) + dist_info.mkdir() - with io.open(osp.join(dist_info, 'WHEEL'), 'w', encoding='utf-8') as f: + with dist_info.joinpath('WHEEL').open('w', encoding='utf-8') as f: _write_wheel_file(f, supports_py2=metadata.supports_py2) - with io.open(osp.join(dist_info, 'METADATA'), 'w', encoding='utf-8') as f: + with dist_info.joinpath('METADATA').open('w', encoding='utf-8') as f: metadata.write_metadata_file(f) if ini_info.entrypoints: - with io.open(osp.join(dist_info, 'entry_points.txt'), 'w', encoding='utf-8') as f: + with dist_info.joinpath('entry_points.txt').open('w', encoding='utf-8') as f: write_entry_points(ini_info.entrypoints, f) - return osp.basename(dist_info) + return Path(dist_info).name # Metadata for editable are the same as for a wheel prepare_metadata_for_build_editable = prepare_metadata_for_build_wheel def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): """Builds a wheel, places it in wheel_directory""" - info = make_wheel_in(pyproj_toml, Path(wheel_directory)) + info = make_wheel_in(pyproj_toml, wheel_directory) return info.file.name def build_editable(wheel_directory, config_settings=None, metadata_directory=None): """Builds an "editable" wheel, places it in wheel_directory""" - info = make_wheel_in(pyproj_toml, Path(wheel_directory), editable=True) + info = make_wheel_in(pyproj_toml, wheel_directory, editable=True) return info.file.name def build_sdist(sdist_directory, config_settings=None): """Builds an sdist, places it in sdist_directory""" - path = SdistBuilder.from_ini_path(pyproj_toml).build(Path(sdist_directory)) + path = SdistBuilder.from_ini_path(pyproj_toml).build(sdist_directory) return path.name diff --git a/flit_core/flit_core/common.py b/flit_core/flit_core/common.py index dd009f52..33f7a4b1 100644 --- a/flit_core/flit_core/common.py +++ b/flit_core/flit_core/common.py @@ -21,11 +21,11 @@ def __init__(self, name, directory=Path()): self.name = name # It must exist either as a .py file or a directory, but not both - name_as_path = name.replace('.', os.sep) - pkg_dir = directory / name_as_path - py_file = directory / (name_as_path+'.py') - src_pkg_dir = directory / 'src' / name_as_path - src_py_file = directory / 'src' / (name_as_path+'.py') + name_as_path = Path(name.replace('.', os.sep)) + pkg_dir = directory.joinpath(name_as_path) + py_file = directory.joinpath(name_as_path).with_suffix('.py') + src_pkg_dir = directory.joinpath('src').joinpath(name_as_path) + src_py_file = directory.joinpath('src').joinpath(name_as_path).with_suffix('.py') existing = set() if pkg_dir.is_dir(): @@ -57,7 +57,7 @@ def __init__(self, name, directory=Path()): elif not existing: raise ValueError("No file/folder found for module {}".format(name)) - self.source_dir = directory / self.prefix + self.source_dir = directory.joinpath(self.prefix) if '.' in name: self.namespace_package_name = name.rpartition('.')[0] @@ -66,7 +66,7 @@ def __init__(self, name, directory=Path()): @property def file(self): if self.is_package: - return self.path / '__init__.py' + return self.path.joinpath('__init__.py') else: return self.path @@ -76,24 +76,20 @@ def iter_files(self): Yields absolute paths - caller may want to make them relative. Excludes any __pycache__ and *.pyc files. """ - def _include(path): - name = os.path.basename(path) - if (name == '__pycache__') or name.endswith('.pyc'): - return False - return True + def _walk(path): + files = [] + for path in path.iterdir(): + if path.is_file() and path.suffix != '.pyc': + files.append(path) + if path.is_dir() and path.name != '__pycache__': + files.extend(_walk(path)) + return files if self.is_package: # Ensure we sort all files and directories so the order is stable - for dirpath, dirs, files in os.walk(str(self.path)): - for file in sorted(files): - full_path = os.path.join(dirpath, file) - if _include(full_path): - yield full_path - - dirs[:] = [d for d in sorted(dirs) if _include(d)] - + return sorted(_walk(self.path)) else: - yield str(self.path) + return [self.path] class ProblemInModule(ValueError): pass class NoDocstringError(ProblemInModule): pass diff --git a/flit_core/flit_core/config.py b/flit_core/flit_core/config.py index b0b6ddbd..be514567 100644 --- a/flit_core/flit_core/config.py +++ b/flit_core/flit_core/config.py @@ -3,8 +3,7 @@ import errno import logging import os -import os.path as osp -from pathlib import Path +from pathlib import Path, PurePosixPath import re from .vendor import tomli @@ -177,6 +176,37 @@ def _flatten(d, prefix): res.update(_flatten(v, k)) return res +def posix_normpath(path): + """Normalize path, eliminating double slashes, etc.""" + path = str(path) + sep = '/' + empty = '' + dot = '.' + dotdot = '..' + if path == empty: + return dot + initial_slashes = path.startswith(sep) + # POSIX allows one or two initial slashes, but treats three or more + # as single slash. + # (see http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13) + if (initial_slashes and + path.startswith(sep*2) and not path.startswith(sep*3)): + initial_slashes = 2 + comps = path.split(sep) + new_comps = [] + for comp in comps: + if comp in (empty, dot): + continue + if (comp != dotdot or (not initial_slashes and not new_comps) or + (new_comps and new_comps[-1] == dotdot)): + new_comps.append(comp) + elif new_comps: + new_comps.pop() + comps = new_comps + path = sep.join(comps) + if initial_slashes: + path = sep*initial_slashes + path + return PurePosixPath(path or dot) def _check_glob_patterns(pats, clude): """Check and normalise glob patterns for sdist include/exclude""" @@ -201,18 +231,18 @@ def _check_glob_patterns(pats, clude): .format(clude, p) ) - normp = osp.normpath(p) + normp = posix_normpath(PurePosixPath(p)) - if osp.isabs(normp): + if normp.is_absolute(): raise ConfigError( '{} pattern {!r} is an absolute path'.format(clude, p) ) - if osp.normpath(p).startswith('..' + os.sep): + if PurePosixPath(normp.parts[0]) == PurePosixPath('..'): raise ConfigError( '{} pattern {!r} points out of the directory containing pyproject.toml' .format(clude, p) ) - normed.append(normp) + normed.append(str(normp)) return normed @@ -243,10 +273,10 @@ def add_scripts(self, scripts_dict): def description_from_file(rel_path: str, proj_dir: Path, guess_mimetype=True): - if osp.isabs(rel_path): + if PurePosixPath(rel_path).is_absolute(): raise ConfigError("Readme path must be relative") - desc_path = proj_dir / rel_path + desc_path = proj_dir.joinpath(rel_path) try: with desc_path.open('r', encoding='utf-8') as f: raw_desc = f.read() diff --git a/flit_core/flit_core/sdist.py b/flit_core/flit_core/sdist.py index 80ccf25d..55a00d3b 100644 --- a/flit_core/flit_core/sdist.py +++ b/flit_core/flit_core/sdist.py @@ -1,11 +1,9 @@ from collections import defaultdict from copy import copy -from glob import glob from gzip import GzipFile import io import logging import os -import os.path as osp from pathlib import Path from posixpath import join as pjoin import tarfile @@ -43,9 +41,9 @@ def __init__(self, patterns, basedir): self.files = set() for pattern in patterns: - for path in sorted(glob(osp.join(basedir, pattern))): - rel = osp.relpath(path, basedir) - if osp.isdir(path): + for path in self.basedir.glob(pattern): + rel = path.relative_to(basedir) + if rel.is_dir(): self.dirs.add(rel) else: self.files.add(rel) @@ -54,14 +52,14 @@ def match_file(self, rel_path): if rel_path in self.files: return True - return any(rel_path.startswith(d + os.sep) for d in self.dirs) + return any(d in rel_path.parents for d in self.dirs) def match_dir(self, rel_path): if rel_path in self.dirs: return True # Check if it's a subdirectory of any directory in the list - return any(rel_path.startswith(d + os.sep) for d in self.dirs) + return any(d in rel_path.parents for d in self.dirs) class SdistBuilder: @@ -75,12 +73,12 @@ def __init__(self, module, metadata, cfgdir, reqs_by_extra, entrypoints, extra_files, include_patterns=(), exclude_patterns=()): self.module = module self.metadata = metadata - self.cfgdir = cfgdir + self.cfgdir = Path(cfgdir) self.reqs_by_extra = reqs_by_extra self.entrypoints = entrypoints - self.extra_files = extra_files - self.includes = FilePatterns(include_patterns, str(cfgdir)) - self.excludes = FilePatterns(exclude_patterns, str(cfgdir)) + self.extra_files = [Path(p) for p in extra_files] + self.includes = FilePatterns(include_patterns, cfgdir) + self.excludes = FilePatterns(exclude_patterns, cfgdir) @classmethod def from_ini_path(cls, ini_path: Path): @@ -112,39 +110,30 @@ def select_files(self): This is overridden in flit itself to use information from a VCS to include tests, docs, etc. for a 'gold standard' sdist. """ - cfgdir_s = str(self.cfgdir) return [ - osp.relpath(p, cfgdir_s) for p in self.module.iter_files() + p.relative_to(self.cfgdir) for p in self.module.iter_files() ] + self.extra_files def apply_includes_excludes(self, files): - cfgdir_s = str(self.cfgdir) - files = {f for f in files if not self.excludes.match_file(f)} + files = {Path(f) for f in files if not self.excludes.match_file(Path(f))} for f_rel in self.includes.files: if not self.excludes.match_file(f_rel): files.add(f_rel) for rel_d in self.includes.dirs: - for dirpath, dirs, dfiles in os.walk(osp.join(cfgdir_s, rel_d)): - for file in dfiles: - f_abs = osp.join(dirpath, file) - f_rel = osp.relpath(f_abs, cfgdir_s) - if not self.excludes.match_file(f_rel): - files.add(f_rel) - - # Filter subdirectories before os.walk scans them - dirs[:] = [d for d in dirs if not self.excludes.match_dir( - osp.relpath(osp.join(dirpath, d), cfgdir_s) - )] + for abs_path in self.cfgdir.joinpath(rel_d).glob('**/*'): + path = abs_path.relative_to(self.cfgdir) + if not self.excludes.match_file(path): + files.add(path) crucial_files = set( - self.extra_files + [str(self.module.file.relative_to(self.cfgdir))] + self.extra_files + [self.module.file.relative_to(self.cfgdir)] ) missing_crucial = crucial_files - files if missing_crucial: raise Exception("Crucial files were excluded from the sdist: {}" - .format(", ".join(missing_crucial))) + .format(", ".join(str(m) for m in missing_crucial))) return sorted(files) @@ -157,26 +146,27 @@ def dir_name(self): return '{}-{}'.format(self.metadata.name, self.metadata.version) def build(self, target_dir, gen_setup_py=True): - os.makedirs(str(target_dir), exist_ok=True) - target = target_dir / '{}-{}.tar.gz'.format( + target_dir = Path(target_dir) + target_dir.mkdir(exist_ok=True) + target = target_dir.joinpath('{}-{}.tar.gz'.format( self.metadata.name, self.metadata.version - ) + )) source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH', '') mtime = int(source_date_epoch) if source_date_epoch else None - gz = GzipFile(str(target), mode='wb', mtime=mtime) - tf = tarfile.TarFile(str(target), mode='w', fileobj=gz, + gz = GzipFile(target, mode='wb', mtime=mtime) + tf = tarfile.TarFile(target, mode='w', fileobj=gz, format=tarfile.PAX_FORMAT) try: files_to_add = self.apply_includes_excludes(self.select_files()) for relpath in files_to_add: - path = str(self.cfgdir / relpath) - ti = tf.gettarinfo(path, arcname=pjoin(self.dir_name, relpath)) + path = self.cfgdir.joinpath(relpath) + ti = tf.gettarinfo(str(path), arcname=pjoin(self.dir_name, relpath)) ti = clean_tarinfo(ti, mtime) if ti.isreg(): - with open(path, 'rb') as f: + with path.open('rb') as f: tf.addfile(ti, f) else: tf.addfile(ti) # Symlinks & ? diff --git a/flit_core/flit_core/tests/test_sdist.py b/flit_core/flit_core/tests/test_sdist.py index ca29c4e9..40ee32c8 100644 --- a/flit_core/flit_core/tests/test_sdist.py +++ b/flit_core/flit_core/tests/test_sdist.py @@ -1,5 +1,4 @@ from io import BytesIO -import os.path as osp from pathlib import Path import tarfile from testpath import assert_isfile @@ -46,6 +45,6 @@ def test_include_exclude(): ) files = builder.apply_includes_excludes(builder.select_files()) - assert osp.join('doc', 'test.rst') in files - assert osp.join('doc', 'test.txt') not in files - assert osp.join('doc', 'subdir', 'test.txt') in files + assert Path('doc').joinpath('test.rst') in files + assert Path('doc').joinpath('test.txt') not in files + assert Path('doc').joinpath('subdir').joinpath('test.txt') in files diff --git a/flit_core/flit_core/wheel.py b/flit_core/flit_core/wheel.py index 8c87451e..24426cfa 100644 --- a/flit_core/flit_core/wheel.py +++ b/flit_core/flit_core/wheel.py @@ -5,9 +5,8 @@ import io import logging import os -import os.path as osp import stat -import tempfile +from pathlib import Path from types import SimpleNamespace from typing import Optional import zipfile @@ -57,7 +56,7 @@ def zip_timestamp_from_env() -> Optional[tuple]: class WheelBuilder: - def __init__(self, directory, module, metadata, entrypoints, target_fp): + def __init__(self, directory, module, metadata, entrypoints, target_buffer): """Build a wheel from a module/package """ self.directory = directory @@ -69,11 +68,11 @@ def __init__(self, directory, module, metadata, entrypoints, target_fp): self.source_time_stamp = zip_timestamp_from_env() # Open the zip file ready to write - self.wheel_zip = zipfile.ZipFile(target_fp, 'w', + self.wheel_zip = zipfile.ZipFile(target_buffer, 'w', compression=zipfile.ZIP_DEFLATED) @classmethod - def from_ini_path(cls, ini_path, target_fp): + def from_ini_path(cls, ini_path, target_buffer): # Local import so bootstrapping doesn't try to load toml from .config import read_flit_config directory = ini_path.parent @@ -81,7 +80,7 @@ def from_ini_path(cls, ini_path, target_fp): entrypoints = ini_info.entrypoints module = common.Module(ini_info.module, directory) metadata = common.make_metadata(module, ini_info) - return cls(directory, module, metadata, entrypoints, target_fp) + return cls(directory, module, metadata, entrypoints, target_buffer) @property def dist_info(self): @@ -150,10 +149,9 @@ def _write_to_zip(self, rel_path, mode=0o644): def copy_module(self): log.info('Copying package file(s) from %s', self.module.path) - source_dir = str(self.module.source_dir) for full_path in self.module.iter_files(): - rel_path = osp.relpath(full_path, source_dir) + rel_path = full_path.relative_to(self.module.source_dir) self._add_file(full_path, rel_path) def add_pth(self): @@ -201,17 +199,14 @@ def build(self, editable=False): def make_wheel_in(ini_path, wheel_directory, editable=False): # We don't know the final filename until metadata is loaded, so write to # a temporary_file, and rename it afterwards. - (fd, temp_path) = tempfile.mkstemp(suffix='.whl', dir=str(wheel_directory)) - try: - with io.open(fd, 'w+b') as fp: - wb = WheelBuilder.from_ini_path(ini_path, fp) - wb.build(editable) - - wheel_path = wheel_directory / wb.wheel_filename - os.replace(temp_path, str(wheel_path)) - except: - os.unlink(temp_path) - raise + wheel_directory = Path(wheel_directory) + with io.BytesIO() as buffer: + wb = WheelBuilder.from_ini_path(ini_path, buffer) + wb.build(editable) + + wheel_path = wheel_directory.joinpath(wb.wheel_filename) + with wheel_path.open(mode='w+b') as wp: + wp.write(buffer.getvalue()) log.info("Built wheel: %s", wheel_path) return SimpleNamespace(builder=wb, file=wheel_path)