diff --git a/.github/workflows/python-linters.yml b/.github/workflows/python-linters.yml new file mode 100644 index 00000000000..bf3308b0223 --- /dev/null +++ b/.github/workflows/python-linters.yml @@ -0,0 +1,46 @@ +name: Code quality + +on: + push: + pull_request: + schedule: + # Run every Friday at 18:02 UTC + # https://crontab.guru/#2_18_*_*_5 + - cron: 2 18 * * 5 + +jobs: + linters: + name: 🤖 + runs-on: ${{ matrix.os }} + strategy: + # max-parallel: 5 + matrix: + os: + - ubuntu-18.04 + env: + - TOXENV: docs + - TOXENV: lint + - TOXENV: lint-py2 + PYTHON_VERSION: 2.7 + - TOXENV: mypy + - TOXENV: packaging + steps: + - uses: actions/checkout@master + - name: Set up Python ${{ matrix.env.PYTHON_VERSION || 3.7 }} + uses: actions/setup-python@v1 + with: + version: ${{ matrix.env.PYTHON_VERSION || 3.7 }} + - name: Pre-configure global Git settings + run: >- + tools/travis/setup.sh + - name: Update setuptools and tox dependencies + run: >- + tools/travis/install.sh + - name: 'Initialize tox envs: ${{ matrix.env.TOXENV }}' + run: >- + python -m tox --notest --skip-missing-interpreters false + env: ${{ matrix.env }} + - name: Test with tox + run: >- + python -m tox + env: ${{ matrix.env }} diff --git a/docs/html/development/index.rst b/docs/html/development/index.rst index 7d10230e00c..53fefc9e186 100644 --- a/docs/html/development/index.rst +++ b/docs/html/development/index.rst @@ -14,6 +14,7 @@ or the `pypa-dev mailing list`_, to ask questions or get involved. getting-started contributing + issue-triage architecture/index release-process vendoring-policy diff --git a/docs/html/development/issue-triage.rst b/docs/html/development/issue-triage.rst new file mode 100644 index 00000000000..621b9e6a453 --- /dev/null +++ b/docs/html/development/issue-triage.rst @@ -0,0 +1,25 @@ +.. note:: + This section of the documentation is currently being written. pip + developers welcome your help to complete this documentation. If you're + interested in helping out, please let us know in the + `tracking issue `__. + +============ +Issue Triage +============ + +This serves as an introduction to issue tracking in pip as well as +how to help triage reported issues. + + +Issue Tracker +************* + +The `pip issue tracker `__ is hosted on +GitHub alongside the project. + +Currently, the issue tracker is used for bugs, feature requests, and general +user support. + +In the pip issue tracker, we make use of labels and milestones to organize and +track work. diff --git a/docs/html/reference/pip_install.rst b/docs/html/reference/pip_install.rst index f56dc9a0e9f..fa475c462c8 100644 --- a/docs/html/reference/pip_install.rst +++ b/docs/html/reference/pip_install.rst @@ -244,8 +244,7 @@ pip supports installing from a package index using a :term:`requirement specifier `. Generally speaking, a requirement specifier is composed of a project name followed by optional :term:`version specifiers `. :pep:`508` contains a full specification -of the format of a requirement (pip does not support the ``url_req`` form -of specifier at this time). +of the format of a requirement. Some examples: @@ -265,6 +264,13 @@ Since version 6.0, pip also supports specifiers containing `environment markers SomeProject ==5.4 ; python_version < '2.7' SomeProject; sys_platform == 'win32' +Since version 19.1, pip also supports `direct references +`__ like so: + + :: + + SomeProject @ file:///somewhere/... + Environment markers are supported in the command line and in requirements files. .. note:: @@ -880,6 +886,14 @@ Examples $ pip install http://my.package.repo/SomePackage-1.0.4.zip +#. Install a particular source archive file following :pep:`440` direct references. + + :: + + $ pip install SomeProject==1.0.4@http://my.package.repo//SomeProject-1.2.3-py33-none-any.whl + $ pip install "SomeProject==1.0.4 @ http://my.package.repo//SomeProject-1.2.3-py33-none-any.whl" + + #. Install from alternative package repositories. Install from a different index, and not `PyPI`_ :: diff --git a/news/4910.bugfix b/news/4910.bugfix new file mode 100644 index 00000000000..e829dfc7467 --- /dev/null +++ b/news/4910.bugfix @@ -0,0 +1 @@ +Fix ``rmtree_errorhandler`` to skip non-existing directories. diff --git a/news/6202.bugfix b/news/6202.bugfix new file mode 100644 index 00000000000..03184fa8d93 --- /dev/null +++ b/news/6202.bugfix @@ -0,0 +1,2 @@ +Fix requirement line parser to correctly handle PEP 440 requirements with a URL +pointing to an archive file. diff --git a/news/6892.bugfix b/news/6892.bugfix new file mode 100644 index 00000000000..763f2520b47 --- /dev/null +++ b/news/6892.bugfix @@ -0,0 +1,2 @@ +Correctly uninstall symlinks that were installed in a virtualenv, +by tools such as ``flit install --symlink``. \ No newline at end of file diff --git a/news/6952-gh-actions--linters.trivial b/news/6952-gh-actions--linters.trivial new file mode 100644 index 00000000000..194e39025c0 --- /dev/null +++ b/news/6952-gh-actions--linters.trivial @@ -0,0 +1 @@ +Add a GitHub Actions workflow running all linters. diff --git a/news/6954.bugfix b/news/6954.bugfix new file mode 100644 index 00000000000..8f6f67109cb --- /dev/null +++ b/news/6954.bugfix @@ -0,0 +1 @@ +Don't use hardlinks for locking selfcheck state file. diff --git a/news/6991.bugfix b/news/6991.bugfix new file mode 100644 index 00000000000..db5904cdd42 --- /dev/null +++ b/news/6991.bugfix @@ -0,0 +1 @@ + Ignore "require_virtualenv" in `pip config` diff --git a/news/7037.removal b/news/7037.removal new file mode 100644 index 00000000000..577a02f5e46 --- /dev/null +++ b/news/7037.removal @@ -0,0 +1,2 @@ +Remove undocumented support for http:// requirements pointing to SVN +repositories. diff --git a/news/deprecated-yield-fixture.trivial b/news/deprecated-yield-fixture.trivial new file mode 100644 index 00000000000..552f49c2ce1 --- /dev/null +++ b/news/deprecated-yield-fixture.trivial @@ -0,0 +1,2 @@ +Use normal ``fixture`` instead of ``yield_fixture``. +It's been deprecated in pytest since 2.10 version. diff --git a/news/lockfile.vendor b/news/lockfile.vendor new file mode 100644 index 00000000000..3d58fa13807 --- /dev/null +++ b/news/lockfile.vendor @@ -0,0 +1 @@ +Remove Lockfile as a vendored dependency. diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index 155211903b1..d8a6ebd54b6 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -24,10 +24,8 @@ from pip._internal.locations import USER_CACHE_DIR, get_src_prefix from pip._internal.models.format_control import FormatControl from pip._internal.models.index import PyPI -from pip._internal.models.search_scope import SearchScope from pip._internal.models.target_python import TargetPython from pip._internal.utils.hashes import STRONG_HASHES -from pip._internal.utils.misc import redact_auth_from_url from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.ui import BAR_TYPES @@ -359,30 +357,6 @@ def find_links(): ) -def make_search_scope(options, suppress_no_index=False): - # type: (Values, bool) -> SearchScope - """ - :param suppress_no_index: Whether to ignore the --no-index option - when constructing the SearchScope object. - """ - index_urls = [options.index_url] + options.extra_index_urls - if options.no_index and not suppress_no_index: - logger.debug( - 'Ignoring indexes: %s', - ','.join(redact_auth_from_url(url) for url in index_urls), - ) - index_urls = [] - - # Make sure find_links is a list before passing to create(). - find_links = options.find_links or [] - - search_scope = SearchScope.create( - find_links=find_links, index_urls=index_urls, - ) - - return search_scope - - def trusted_host(): # type: () -> Option return Option( diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index e1fd495e4c9..9a2d4196f6e 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -9,7 +9,6 @@ from functools import partial from pip._internal.cli.base_command import Command -from pip._internal.cli.cmdoptions import make_search_scope from pip._internal.cli.command_context import CommandContextMixIn from pip._internal.download import PipSession from pip._internal.exceptions import CommandError @@ -24,7 +23,7 @@ ) from pip._internal.req.req_file import parse_requirements from pip._internal.utils.misc import normalize_path -from pip._internal.utils.outdated import pip_version_check +from pip._internal.utils.outdated import make_link_collector, pip_version_check from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: @@ -283,7 +282,7 @@ def _build_package_finder( :param ignore_requires_python: Whether to ignore incompatible "Requires-Python" values in links. Defaults to False. """ - search_scope = make_search_scope(options) + link_collector = make_link_collector(session, options=options) selection_prefs = SelectionPreferences( allow_yanked=True, format_control=options.format_control, @@ -293,8 +292,7 @@ def _build_package_finder( ) return PackageFinder.create( - search_scope=search_scope, + link_collector=link_collector, selection_prefs=selection_prefs, - session=session, target_python=target_python, ) diff --git a/src/pip/_internal/collector.py b/src/pip/_internal/collector.py new file mode 100644 index 00000000000..443e22a7f5b --- /dev/null +++ b/src/pip/_internal/collector.py @@ -0,0 +1,532 @@ +""" +The main purpose of this module is to expose LinkCollector.collect_links(). +""" + +import cgi +import itertools +import logging +import mimetypes +import os + +from pip._vendor import html5lib, requests +from pip._vendor.distlib.compat import unescape +from pip._vendor.requests.exceptions import HTTPError, RetryError, SSLError +from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.six.moves.urllib import request as urllib_request + +from pip._internal.models.link import Link +from pip._internal.utils.filetypes import ARCHIVE_EXTENSIONS +from pip._internal.utils.misc import path_to_url, redact_auth_from_url +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import url_to_path +from pip._internal.vcs import is_url, vcs + +if MYPY_CHECK_RUNNING: + from typing import ( + Callable, Dict, Iterable, List, MutableMapping, Optional, Sequence, + Set, Tuple, Union, + ) + import xml.etree.ElementTree + + from pip._vendor.requests import Response + + from pip._internal.models.search_scope import SearchScope + from pip._internal.download import PipSession + + HTMLElement = xml.etree.ElementTree.Element + ResponseHeaders = MutableMapping[str, str] + + +logger = logging.getLogger(__name__) + + +def _match_vcs_scheme(url): + # type: (str) -> Optional[str] + """Look for VCS schemes in the URL. + + Returns the matched VCS scheme, or None if there's no match. + """ + for scheme in vcs.schemes: + if url.lower().startswith(scheme) and url[len(scheme)] in '+:': + return scheme + return None + + +def _is_url_like_archive(url): + # type: (str) -> bool + """Return whether the URL looks like an archive. + """ + filename = Link(url).filename + for bad_ext in ARCHIVE_EXTENSIONS: + if filename.endswith(bad_ext): + return True + return False + + +class _NotHTML(Exception): + def __init__(self, content_type, request_desc): + # type: (str, str) -> None + super(_NotHTML, self).__init__(content_type, request_desc) + self.content_type = content_type + self.request_desc = request_desc + + +def _ensure_html_header(response): + # type: (Response) -> None + """Check the Content-Type header to ensure the response contains HTML. + + Raises `_NotHTML` if the content type is not text/html. + """ + content_type = response.headers.get("Content-Type", "") + if not content_type.lower().startswith("text/html"): + raise _NotHTML(content_type, response.request.method) + + +class _NotHTTP(Exception): + pass + + +def _ensure_html_response(url, session): + # type: (str, PipSession) -> None + """Send a HEAD request to the URL, and ensure the response contains HTML. + + Raises `_NotHTTP` if the URL is not available for a HEAD request, or + `_NotHTML` if the content type is not text/html. + """ + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) + if scheme not in {'http', 'https'}: + raise _NotHTTP() + + resp = session.head(url, allow_redirects=True) + resp.raise_for_status() + + _ensure_html_header(resp) + + +def _get_html_response(url, session): + # type: (str, PipSession) -> Response + """Access an HTML page with GET, and return the response. + + This consists of three parts: + + 1. If the URL looks suspiciously like an archive, send a HEAD first to + check the Content-Type is HTML, to avoid downloading a large file. + Raise `_NotHTTP` if the content type cannot be determined, or + `_NotHTML` if it is not HTML. + 2. Actually perform the request. Raise HTTP exceptions on network failures. + 3. Check the Content-Type header to make sure we got HTML, and raise + `_NotHTML` otherwise. + """ + if _is_url_like_archive(url): + _ensure_html_response(url, session=session) + + logger.debug('Getting page %s', redact_auth_from_url(url)) + + resp = session.get( + url, + headers={ + "Accept": "text/html", + # We don't want to blindly returned cached data for + # /simple/, because authors generally expecting that + # twine upload && pip install will function, but if + # they've done a pip install in the last ~10 minutes + # it won't. Thus by setting this to zero we will not + # blindly use any cached data, however the benefit of + # using max-age=0 instead of no-cache, is that we will + # still support conditional requests, so we will still + # minimize traffic sent in cases where the page hasn't + # changed at all, we will just always incur the round + # trip for the conditional GET now instead of only + # once per 10 minutes. + # For more information, please see pypa/pip#5670. + "Cache-Control": "max-age=0", + }, + ) + resp.raise_for_status() + + # The check for archives above only works if the url ends with + # something that looks like an archive. However that is not a + # requirement of an url. Unless we issue a HEAD request on every + # url we cannot know ahead of time for sure if something is HTML + # or not. However we can check after we've downloaded it. + _ensure_html_header(resp) + + return resp + + +def _get_encoding_from_headers(headers): + # type: (Optional[ResponseHeaders]) -> Optional[str] + """Determine if we have any encoding information in our headers. + """ + if headers and "Content-Type" in headers: + content_type, params = cgi.parse_header(headers["Content-Type"]) + if "charset" in params: + return params['charset'] + return None + + +def _determine_base_url(document, page_url): + # type: (HTMLElement, str) -> str + """Determine the HTML document's base URL. + + This looks for a ```` tag in the HTML document. If present, its href + attribute denotes the base URL of anchor tags in the document. If there is + no such tag (or if it does not have a valid href attribute), the HTML + file's URL is used as the base URL. + + :param document: An HTML document representation. The current + implementation expects the result of ``html5lib.parse()``. + :param page_url: The URL of the HTML document. + """ + for base in document.findall(".//base"): + href = base.get("href") + if href is not None: + return href + return page_url + + +def _clean_link(url): + # type: (str) -> str + """Makes sure a link is fully encoded. That is, if a ' ' shows up in + the link, it will be rewritten to %20 (while not over-quoting + % or other characters).""" + # Split the URL into parts according to the general structure + # `scheme://netloc/path;parameters?query#fragment`. Note that the + # `netloc` can be empty and the URI will then refer to a local + # filesystem path. + result = urllib_parse.urlparse(url) + # In both cases below we unquote prior to quoting to make sure + # nothing is double quoted. + if result.netloc == "": + # On Windows the path part might contain a drive letter which + # should not be quoted. On Linux where drive letters do not + # exist, the colon should be quoted. We rely on urllib.request + # to do the right thing here. + path = urllib_request.pathname2url( + urllib_request.url2pathname(result.path)) + else: + # In addition to the `/` character we protect `@` so that + # revision strings in VCS URLs are properly parsed. + path = urllib_parse.quote(urllib_parse.unquote(result.path), safe="/@") + return urllib_parse.urlunparse(result._replace(path=path)) + + +def _create_link_from_element( + anchor, # type: HTMLElement + page_url, # type: str + base_url, # type: str +): + # type: (...) -> Optional[Link] + """ + Convert an anchor element in a simple repository page to a Link. + """ + href = anchor.get("href") + if not href: + return None + + url = _clean_link(urllib_parse.urljoin(base_url, href)) + pyrequire = anchor.get('data-requires-python') + pyrequire = unescape(pyrequire) if pyrequire else None + + yanked_reason = anchor.get('data-yanked') + if yanked_reason: + # This is a unicode string in Python 2 (and 3). + yanked_reason = unescape(yanked_reason) + + link = Link( + url, + comes_from=page_url, + requires_python=pyrequire, + yanked_reason=yanked_reason, + ) + + return link + + +def parse_links( + html, # type: bytes + encoding, # type: Optional[str] + url, # type: str +): + # type: (...) -> Iterable[Link] + """ + Parse an HTML document, and yield its anchor elements as Link objects. + + :param url: the URL from which the HTML was downloaded. + """ + document = html5lib.parse( + html, + transport_encoding=encoding, + namespaceHTMLElements=False, + ) + base_url = _determine_base_url(document, url) + for anchor in document.findall(".//a"): + link = _create_link_from_element( + anchor, + page_url=url, + base_url=base_url, + ) + if link is None: + continue + yield link + + +class HTMLPage(object): + """Represents one page, along with its URL""" + + def __init__(self, content, url, headers=None): + # type: (bytes, str, ResponseHeaders) -> None + self.content = content + self.url = url + self.headers = headers + + def __str__(self): + return redact_auth_from_url(self.url) + + def iter_links(self): + # type: () -> Iterable[Link] + """Yields all links in the page""" + encoding = _get_encoding_from_headers(self.headers) + for link in parse_links(self.content, encoding=encoding, url=self.url): + yield link + + +def _handle_get_page_fail( + link, # type: Link + reason, # type: Union[str, Exception] + meth=None # type: Optional[Callable[..., None]] +): + # type: (...) -> None + if meth is None: + meth = logger.debug + meth("Could not fetch URL %s: %s - skipping", link, reason) + + +def _get_html_page(link, session=None): + # type: (Link, Optional[PipSession]) -> Optional[HTMLPage] + if session is None: + raise TypeError( + "_get_html_page() missing 1 required keyword argument: 'session'" + ) + + url = link.url.split('#', 1)[0] + + # Check for VCS schemes that do not support lookup as web pages. + vcs_scheme = _match_vcs_scheme(url) + if vcs_scheme: + logger.debug('Cannot look at %s URL %s', vcs_scheme, link) + return None + + # Tack index.html onto file:// URLs that point to directories + scheme, _, path, _, _, _ = urllib_parse.urlparse(url) + if (scheme == 'file' and os.path.isdir(urllib_request.url2pathname(path))): + # add trailing slash if not present so urljoin doesn't trim + # final segment + if not url.endswith('/'): + url += '/' + url = urllib_parse.urljoin(url, 'index.html') + logger.debug(' file: URL is directory, getting %s', url) + + try: + resp = _get_html_response(url, session=session) + except _NotHTTP: + logger.debug( + 'Skipping page %s because it looks like an archive, and cannot ' + 'be checked by HEAD.', link, + ) + except _NotHTML as exc: + logger.debug( + 'Skipping page %s because the %s request got Content-Type: %s', + link, exc.request_desc, exc.content_type, + ) + except HTTPError as exc: + _handle_get_page_fail(link, exc) + except RetryError as exc: + _handle_get_page_fail(link, exc) + except SSLError as exc: + reason = "There was a problem confirming the ssl certificate: " + reason += str(exc) + _handle_get_page_fail(link, reason, meth=logger.info) + except requests.ConnectionError as exc: + _handle_get_page_fail(link, "connection error: %s" % exc) + except requests.Timeout: + _handle_get_page_fail(link, "timed out") + else: + return HTMLPage(resp.content, resp.url, resp.headers) + return None + + +def group_locations(locations, expand_dir=False): + # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] + """ + Divide a list of locations into two groups: "files" (archives) and "urls." + + :return: A pair of lists (files, urls). + """ + files = [] + urls = [] + + # puts the url for the given file path into the appropriate list + def sort_path(path): + url = path_to_url(path) + if mimetypes.guess_type(url, strict=False)[0] == 'text/html': + urls.append(url) + else: + files.append(url) + + for url in locations: + + is_local_path = os.path.exists(url) + is_file_url = url.startswith('file:') + + if is_local_path or is_file_url: + if is_local_path: + path = url + else: + path = url_to_path(url) + if os.path.isdir(path): + if expand_dir: + path = os.path.realpath(path) + for item in os.listdir(path): + sort_path(os.path.join(path, item)) + elif is_file_url: + urls.append(url) + else: + logger.warning( + "Path '{0}' is ignored: " + "it is a directory.".format(path), + ) + elif os.path.isfile(path): + sort_path(path) + else: + logger.warning( + "Url '%s' is ignored: it is neither a file " + "nor a directory.", url, + ) + elif is_url(url): + # Only add url with clear scheme + urls.append(url) + else: + logger.warning( + "Url '%s' is ignored. It is either a non-existing " + "path or lacks a specific scheme.", url, + ) + + return files, urls + + +class CollectedLinks(object): + + """ + Encapsulates all the Link objects collected by a call to + LinkCollector.collect_links(), stored separately as-- + + (1) links from the configured file locations, + (2) links from the configured find_links, and + (3) a dict mapping HTML page url to links from that page. + """ + + def __init__( + self, + files, # type: List[Link] + find_links, # type: List[Link] + pages, # type: Dict[str, List[Link]] + ): + # type: (...) -> None + """ + :param files: Links from file locations. + :param find_links: Links from find_links. + :param pages: A dict mapping HTML page url to links from that page. + """ + self.files = files + self.find_links = find_links + self.pages = pages + + +class LinkCollector(object): + + """ + Responsible for collecting Link objects from all configured locations, + making network requests as needed. + + The class's main method is its collect_links() method. + """ + + def __init__( + self, + session, # type: PipSession + search_scope, # type: SearchScope + ): + # type: (...) -> None + self.search_scope = search_scope + self.session = session + + @property + def find_links(self): + # type: () -> List[str] + return self.search_scope.find_links + + def _get_pages(self, locations): + # type: (Iterable[Link]) -> Iterable[HTMLPage] + """ + Yields (page, page_url) from the given locations, skipping + locations that have errors. + """ + seen = set() # type: Set[Link] + for location in locations: + if location in seen: + continue + seen.add(location) + + page = _get_html_page(location, session=self.session) + if page is None: + continue + + yield page + + def collect_links(self, project_name): + # type: (str) -> CollectedLinks + """Find all available links for the given project name. + + :return: All the Link objects (unfiltered), as a CollectedLinks object. + """ + search_scope = self.search_scope + index_locations = search_scope.get_index_urls_locations(project_name) + index_file_loc, index_url_loc = group_locations(index_locations) + fl_file_loc, fl_url_loc = group_locations( + self.find_links, expand_dir=True, + ) + + file_links = [ + Link(url) for url in itertools.chain(index_file_loc, fl_file_loc) + ] + + # We trust every directly linked archive in find_links + find_link_links = [Link(url, '-f') for url in self.find_links] + + # We trust every url that the user has given us whether it was given + # via --index-url or --find-links. + # We want to filter out anything that does not have a secure origin. + url_locations = [ + link for link in itertools.chain( + (Link(url) for url in index_url_loc), + (Link(url) for url in fl_url_loc), + ) + if self.session.is_secure_origin(link) + ] + + logger.debug('%d location(s) to search for versions of %s:', + len(url_locations), project_name) + + for location in url_locations: + logger.debug('* %s', location) + + pages_links = {} + for page in self._get_pages(url_locations): + pages_links[page.url] = list(page.iter_links()) + + return CollectedLinks( + files=file_links, + find_links=find_link_links, + pages=pages_links, + ) diff --git a/src/pip/_internal/commands/configuration.py b/src/pip/_internal/commands/configuration.py index 4b3fc2baec4..6c3a0729523 100644 --- a/src/pip/_internal/commands/configuration.py +++ b/src/pip/_internal/commands/configuration.py @@ -34,6 +34,7 @@ class ConfigurationCommand(Command): default. """ + ignore_require_venv = True usage = """ %prog [] list %prog [] [--editor ] edit diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index 6b80d58021e..d6e38db8659 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -7,7 +7,6 @@ from pip._vendor.six.moves import zip_longest from pip._internal.cli import cmdoptions -from pip._internal.cli.cmdoptions import make_search_scope from pip._internal.cli.req_command import IndexGroupCommand from pip._internal.exceptions import CommandError from pip._internal.index import PackageFinder @@ -17,6 +16,7 @@ get_installed_distributions, write_output, ) +from pip._internal.utils.outdated import make_link_collector from pip._internal.utils.packaging import get_installer logger = logging.getLogger(__name__) @@ -116,7 +116,7 @@ def _build_package_finder(self, options, session): """ Create a package finder appropriate to this list command. """ - search_scope = make_search_scope(options) + link_collector = make_link_collector(session, options=options) # Pass allow_yanked=False to ignore yanked versions. selection_prefs = SelectionPreferences( @@ -125,9 +125,8 @@ def _build_package_finder(self, options, session): ) return PackageFinder.create( - search_scope=search_scope, + link_collector=link_collector, selection_prefs=selection_prefs, - session=session, ) def run(self, options, args): diff --git a/src/pip/_internal/distributions/source/legacy.py b/src/pip/_internal/distributions/source/legacy.py index e5d9fd4bf80..17f3d169313 100644 --- a/src/pip/_internal/distributions/source/legacy.py +++ b/src/pip/_internal/distributions/source/legacy.py @@ -29,7 +29,13 @@ def prepare_distribution_metadata(self, finder, build_isolation): self.req.load_pyproject_toml() should_isolate = self.req.use_pep517 and build_isolation + if should_isolate: + self._setup_isolation(finder) + self.req.prepare_metadata() + self.req.assert_source_matches_version() + + def _setup_isolation(self, finder): def _raise_conflicts(conflicting_with, conflicting_reqs): raise InstallationError( "Some build dependencies for %s conflict with %s: %s." % ( @@ -37,44 +43,40 @@ def _raise_conflicts(conflicting_with, conflicting_reqs): '%s is incompatible with %s' % (installed, wanted) for installed, wanted in sorted(conflicting)))) - if should_isolate: - # Isolate in a BuildEnvironment and install the build-time - # requirements. - self.req.build_env = BuildEnvironment() - self.req.build_env.install_requirements( - finder, self.req.pyproject_requires, 'overlay', - "Installing build dependencies" + # Isolate in a BuildEnvironment and install the build-time + # requirements. + self.req.build_env = BuildEnvironment() + self.req.build_env.install_requirements( + finder, self.req.pyproject_requires, 'overlay', + "Installing build dependencies" + ) + conflicting, missing = self.req.build_env.check_requirements( + self.req.requirements_to_check + ) + if conflicting: + _raise_conflicts("PEP 517/518 supported requirements", + conflicting) + if missing: + logger.warning( + "Missing build requirements in pyproject.toml for %s.", + self.req, ) - conflicting, missing = self.req.build_env.check_requirements( - self.req.requirements_to_check + logger.warning( + "The project does not specify a build backend, and " + "pip cannot fall back to setuptools without %s.", + " and ".join(map(repr, sorted(missing))) ) - if conflicting: - _raise_conflicts("PEP 517/518 supported requirements", - conflicting) - if missing: - logger.warning( - "Missing build requirements in pyproject.toml for %s.", - self.req, - ) - logger.warning( - "The project does not specify a build backend, and " - "pip cannot fall back to setuptools without %s.", - " and ".join(map(repr, sorted(missing))) - ) - # Install any extra build dependencies that the backend requests. - # This must be done in a second pass, as the pyproject.toml - # dependencies must be installed before we can call the backend. - with self.req.build_env: - # We need to have the env active when calling the hook. - self.req.spin_message = "Getting requirements to build wheel" - reqs = self.req.pep517_backend.get_requires_for_build_wheel() - conflicting, missing = self.req.build_env.check_requirements(reqs) - if conflicting: - _raise_conflicts("the backend dependencies", conflicting) - self.req.build_env.install_requirements( - finder, missing, 'normal', - "Installing backend dependencies" - ) - - self.req.prepare_metadata() - self.req.assert_source_matches_version() + # Install any extra build dependencies that the backend requests. + # This must be done in a second pass, as the pyproject.toml + # dependencies must be installed before we can call the backend. + with self.req.build_env: + # We need to have the env active when calling the hook. + self.req.spin_message = "Getting requirements to build wheel" + reqs = self.req.pep517_backend.get_requires_for_build_wheel() + conflicting, missing = self.req.build_env.check_requirements(reqs) + if conflicting: + _raise_conflicts("the backend dependencies", conflicting) + self.req.build_env.install_requirements( + finder, missing, 'normal', + "Installing backend dependencies" + ) diff --git a/src/pip/_internal/download.py b/src/pip/_internal/download.py index 15d6fd56f08..763dc369ac3 100644 --- a/src/pip/_internal/download.py +++ b/src/pip/_internal/download.py @@ -14,8 +14,8 @@ from pip._vendor import requests, six, urllib3 from pip._vendor.cachecontrol import CacheControlAdapter +from pip._vendor.cachecontrol.cache import BaseCache from pip._vendor.cachecontrol.caches import FileCache -from pip._vendor.lockfile import LockError from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response @@ -26,7 +26,6 @@ # why we ignore the type on this import from pip._vendor.six.moves import xmlrpc_client # type: ignore from pip._vendor.six.moves.urllib import parse as urllib_parse -from pip._vendor.six.moves.urllib import request as urllib_request import pip from pip._internal.exceptions import HashMismatch, InstallationError @@ -34,10 +33,14 @@ # Import ssl from compat so the initial import occurs in only one place. from pip._internal.utils.compat import HAS_TLS, ipaddress, ssl from pip._internal.utils.encoding import auto_decode -from pip._internal.utils.filesystem import check_path_owner, copy2_fixed +from pip._internal.utils.filesystem import ( + adjacent_tmp_file, + check_path_owner, + copy2_fixed, + replace, +) from pip._internal.utils.glibc import libc_ver from pip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, ask, ask_input, ask_password, @@ -46,6 +49,7 @@ build_url_from_netloc, consume, display_path, + ensure_dir, format_size, get_installed_version, hide_url, @@ -56,11 +60,12 @@ rmtree, split_auth_netloc_from_url, splitext, - unpack_file, ) from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.ui import DownloadProgressProvider +from pip._internal.utils.unpacking import unpack_file +from pip._internal.utils.urls import get_url_scheme, url_to_path from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: @@ -101,8 +106,8 @@ __all__ = ['get_file_content', - 'is_url', 'url_to_path', 'path_to_url', - 'is_archive_file', 'unpack_vcs_link', + 'path_to_url', + 'unpack_vcs_link', 'unpack_file_url', 'is_file_url', 'unpack_http_url', 'unpack_url', 'parse_content_disposition', 'sanitize_content_filename'] @@ -533,31 +538,54 @@ def suppressed_cache_errors(): """ try: yield - except (LockError, OSError, IOError): + except (OSError, IOError): pass -class SafeFileCache(FileCache): +class SafeFileCache(BaseCache): """ A file based cache which is safe to use even when the target directory may not be accessible or writable. """ - def __init__(self, directory, *args, **kwargs): + def __init__(self, directory): + # type: (str) -> None assert directory is not None, "Cache directory must not be None." - super(SafeFileCache, self).__init__(directory, *args, **kwargs) - - def get(self, *args, **kwargs): + super(SafeFileCache, self).__init__() + self.directory = directory + + def _get_cache_path(self, name): + # type: (str) -> str + # From cachecontrol.caches.file_cache.FileCache._fn, brought into our + # class for backwards-compatibility and to avoid using a non-public + # method. + hashed = FileCache.encode(name) + parts = list(hashed[:5]) + [hashed] + return os.path.join(self.directory, *parts) + + def get(self, key): + # type: (str) -> Optional[bytes] + path = self._get_cache_path(key) with suppressed_cache_errors(): - return super(SafeFileCache, self).get(*args, **kwargs) + with open(path, 'rb') as f: + return f.read() - def set(self, *args, **kwargs): + def set(self, key, value): + # type: (str, bytes) -> None + path = self._get_cache_path(key) with suppressed_cache_errors(): - return super(SafeFileCache, self).set(*args, **kwargs) + ensure_dir(os.path.dirname(path)) + + with adjacent_tmp_file(path) as f: + f.write(value) - def delete(self, *args, **kwargs): + replace(f.name, path) + + def delete(self, key): + # type: (str) -> None + path = self._get_cache_path(key) with suppressed_cache_errors(): - return super(SafeFileCache, self).delete(*args, **kwargs) + os.remove(path) class InsecureHTTPAdapter(HTTPAdapter): @@ -632,7 +660,7 @@ def __init__(self, *args, **kwargs): # require manual eviction from the cache to fix it. if cache: secure_adapter = CacheControlAdapter( - cache=SafeFileCache(cache, use_dir_lock=True), + cache=SafeFileCache(cache), max_retries=retries, ) else: @@ -787,7 +815,7 @@ def get_file_content(url, comes_from=None, session=None): "get_file_content() missing 1 required keyword argument: 'session'" ) - scheme = _get_url_scheme(url) + scheme = get_url_scheme(url) if scheme in ['http', 'https']: # FIXME: catch some errors @@ -824,57 +852,6 @@ def get_file_content(url, comes_from=None, session=None): _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) -def _get_url_scheme(url): - # type: (Union[str, Text]) -> Optional[Text] - if ':' not in url: - return None - return url.split(':', 1)[0].lower() - - -def is_url(name): - # type: (Union[str, Text]) -> bool - """Returns true if the name looks like a URL""" - scheme = _get_url_scheme(name) - if scheme is None: - return False - return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes - - -def url_to_path(url): - # type: (str) -> str - """ - Convert a file: URL to a path. - """ - assert url.startswith('file:'), ( - "You can only turn file: urls into filenames (not %r)" % url) - - _, netloc, path, _, _ = urllib_parse.urlsplit(url) - - if not netloc or netloc == 'localhost': - # According to RFC 8089, same as empty authority. - netloc = '' - elif sys.platform == 'win32': - # If we have a UNC path, prepend UNC share notation. - netloc = '\\\\' + netloc - else: - raise ValueError( - 'non-local file URIs are not supported on this platform: %r' - % url - ) - - path = urllib_request.url2pathname(netloc + path) - return path - - -def is_archive_file(name): - # type: (str) -> bool - """Return True if `name` is a considered as an archive file.""" - ext = splitext(name)[1].lower() - if ext in ARCHIVE_EXTENSIONS: - return True - return False - - def unpack_vcs_link(link, location): # type: (Link, str) -> None vcs_backend = _get_used_vcs_backend(link) @@ -1076,7 +1053,7 @@ def unpack_http_url( # unpack the archive to the build dir location. even when only # downloading archives, they have to be unpacked to parse dependencies - unpack_file(from_path, location, content_type, link) + unpack_file(from_path, location, content_type) # a download dir is specified; let's copy the archive there if download_dir and not already_downloaded_path: @@ -1172,7 +1149,7 @@ def unpack_file_url( # unpack the archive to the build dir location. even when only downloading # archives, they have to be unpacked to parse dependencies - unpack_file(from_path, location, content_type, link) + unpack_file(from_path, location, content_type) # a download dir is specified and not already downloaded if download_dir and not already_downloaded_path: diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index ed08e0272c2..4db14eb1aa8 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -5,23 +5,13 @@ from __future__ import absolute_import -import cgi -import itertools import logging -import mimetypes -import os import re -from pip._vendor import html5lib, requests -from pip._vendor.distlib.compat import unescape from pip._vendor.packaging import specifiers from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.packaging.version import parse as parse_version -from pip._vendor.requests.exceptions import HTTPError, RetryError, SSLError -from pip._vendor.six.moves.urllib import parse as urllib_parse -from pip._vendor.six.moves.urllib import request as urllib_request -from pip._internal.download import is_url, url_to_path from pip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, @@ -33,30 +23,23 @@ from pip._internal.models.link import Link from pip._internal.models.selection_prefs import SelectionPreferences from pip._internal.models.target_python import TargetPython +from pip._internal.utils.filetypes import WHEEL_EXTENSION from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, - SUPPORTED_EXTENSIONS, - WHEEL_EXTENSION, - build_netloc, - path_to_url, - redact_auth_from_url, -) +from pip._internal.utils.misc import build_netloc from pip._internal.utils.packaging import check_requires_python from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS +from pip._internal.utils.urls import url_to_path from pip._internal.wheel import Wheel if MYPY_CHECK_RUNNING: from typing import ( - Any, Callable, Dict, FrozenSet, Iterable, List, MutableMapping, - Optional, Sequence, Set, Text, Tuple, Union, + Any, FrozenSet, Iterable, List, Optional, Set, Text, Tuple, ) - import xml.etree.ElementTree from pip._vendor.packaging.version import _BaseVersion - from pip._vendor.requests import Response + from pip._internal.collector import LinkCollector from pip._internal.models.search_scope import SearchScope from pip._internal.req import InstallRequirement - from pip._internal.download import PipSession from pip._internal.pep425tags import Pep425Tag from pip._internal.utils.hashes import Hashes @@ -64,7 +47,6 @@ CandidateSortingKey = ( Tuple[int, int, int, _BaseVersion, BuildTag, Optional[int]] ) - HTMLElement = xml.etree.ElementTree.Element __all__ = ['FormatControl', 'BestCandidateResult', 'PackageFinder'] @@ -73,362 +55,6 @@ logger = logging.getLogger(__name__) -def _match_vcs_scheme(url): - # type: (str) -> Optional[str] - """Look for VCS schemes in the URL. - - Returns the matched VCS scheme, or None if there's no match. - """ - from pip._internal.vcs import vcs - for scheme in vcs.schemes: - if url.lower().startswith(scheme) and url[len(scheme)] in '+:': - return scheme - return None - - -def _is_url_like_archive(url): - # type: (str) -> bool - """Return whether the URL looks like an archive. - """ - filename = Link(url).filename - for bad_ext in ARCHIVE_EXTENSIONS: - if filename.endswith(bad_ext): - return True - return False - - -class _NotHTML(Exception): - def __init__(self, content_type, request_desc): - # type: (str, str) -> None - super(_NotHTML, self).__init__(content_type, request_desc) - self.content_type = content_type - self.request_desc = request_desc - - -def _ensure_html_header(response): - # type: (Response) -> None - """Check the Content-Type header to ensure the response contains HTML. - - Raises `_NotHTML` if the content type is not text/html. - """ - content_type = response.headers.get("Content-Type", "") - if not content_type.lower().startswith("text/html"): - raise _NotHTML(content_type, response.request.method) - - -class _NotHTTP(Exception): - pass - - -def _ensure_html_response(url, session): - # type: (str, PipSession) -> None - """Send a HEAD request to the URL, and ensure the response contains HTML. - - Raises `_NotHTTP` if the URL is not available for a HEAD request, or - `_NotHTML` if the content type is not text/html. - """ - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) - if scheme not in {'http', 'https'}: - raise _NotHTTP() - - resp = session.head(url, allow_redirects=True) - resp.raise_for_status() - - _ensure_html_header(resp) - - -def _get_html_response(url, session): - # type: (str, PipSession) -> Response - """Access an HTML page with GET, and return the response. - - This consists of three parts: - - 1. If the URL looks suspiciously like an archive, send a HEAD first to - check the Content-Type is HTML, to avoid downloading a large file. - Raise `_NotHTTP` if the content type cannot be determined, or - `_NotHTML` if it is not HTML. - 2. Actually perform the request. Raise HTTP exceptions on network failures. - 3. Check the Content-Type header to make sure we got HTML, and raise - `_NotHTML` otherwise. - """ - if _is_url_like_archive(url): - _ensure_html_response(url, session=session) - - logger.debug('Getting page %s', redact_auth_from_url(url)) - - resp = session.get( - url, - headers={ - "Accept": "text/html", - # We don't want to blindly returned cached data for - # /simple/, because authors generally expecting that - # twine upload && pip install will function, but if - # they've done a pip install in the last ~10 minutes - # it won't. Thus by setting this to zero we will not - # blindly use any cached data, however the benefit of - # using max-age=0 instead of no-cache, is that we will - # still support conditional requests, so we will still - # minimize traffic sent in cases where the page hasn't - # changed at all, we will just always incur the round - # trip for the conditional GET now instead of only - # once per 10 minutes. - # For more information, please see pypa/pip#5670. - "Cache-Control": "max-age=0", - }, - ) - resp.raise_for_status() - - # The check for archives above only works if the url ends with - # something that looks like an archive. However that is not a - # requirement of an url. Unless we issue a HEAD request on every - # url we cannot know ahead of time for sure if something is HTML - # or not. However we can check after we've downloaded it. - _ensure_html_header(resp) - - return resp - - -def _handle_get_page_fail( - link, # type: Link - reason, # type: Union[str, Exception] - meth=None # type: Optional[Callable[..., None]] -): - # type: (...) -> None - if meth is None: - meth = logger.debug - meth("Could not fetch URL %s: %s - skipping", link, reason) - - -def _get_html_page(link, session=None): - # type: (Link, Optional[PipSession]) -> Optional[HTMLPage] - if session is None: - raise TypeError( - "_get_html_page() missing 1 required keyword argument: 'session'" - ) - - url = link.url.split('#', 1)[0] - - # Check for VCS schemes that do not support lookup as web pages. - vcs_scheme = _match_vcs_scheme(url) - if vcs_scheme: - logger.debug('Cannot look at %s URL %s', vcs_scheme, link) - return None - - # Tack index.html onto file:// URLs that point to directories - scheme, _, path, _, _, _ = urllib_parse.urlparse(url) - if (scheme == 'file' and os.path.isdir(urllib_request.url2pathname(path))): - # add trailing slash if not present so urljoin doesn't trim - # final segment - if not url.endswith('/'): - url += '/' - url = urllib_parse.urljoin(url, 'index.html') - logger.debug(' file: URL is directory, getting %s', url) - - try: - resp = _get_html_response(url, session=session) - except _NotHTTP: - logger.debug( - 'Skipping page %s because it looks like an archive, and cannot ' - 'be checked by HEAD.', link, - ) - except _NotHTML as exc: - logger.debug( - 'Skipping page %s because the %s request got Content-Type: %s', - link, exc.request_desc, exc.content_type, - ) - except HTTPError as exc: - _handle_get_page_fail(link, exc) - except RetryError as exc: - _handle_get_page_fail(link, exc) - except SSLError as exc: - reason = "There was a problem confirming the ssl certificate: " - reason += str(exc) - _handle_get_page_fail(link, reason, meth=logger.info) - except requests.ConnectionError as exc: - _handle_get_page_fail(link, "connection error: %s" % exc) - except requests.Timeout: - _handle_get_page_fail(link, "timed out") - else: - return HTMLPage(resp.content, resp.url, resp.headers) - return None - - -def group_locations(locations, expand_dir=False): - # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] - """ - Divide a list of locations into two groups: "files" (archives) and "urls." - - :return: A pair of lists (files, urls). - """ - files = [] - urls = [] - - # puts the url for the given file path into the appropriate list - def sort_path(path): - url = path_to_url(path) - if mimetypes.guess_type(url, strict=False)[0] == 'text/html': - urls.append(url) - else: - files.append(url) - - for url in locations: - - is_local_path = os.path.exists(url) - is_file_url = url.startswith('file:') - - if is_local_path or is_file_url: - if is_local_path: - path = url - else: - path = url_to_path(url) - if os.path.isdir(path): - if expand_dir: - path = os.path.realpath(path) - for item in os.listdir(path): - sort_path(os.path.join(path, item)) - elif is_file_url: - urls.append(url) - else: - logger.warning( - "Path '{0}' is ignored: " - "it is a directory.".format(path), - ) - elif os.path.isfile(path): - sort_path(path) - else: - logger.warning( - "Url '%s' is ignored: it is neither a file " - "nor a directory.", url, - ) - elif is_url(url): - # Only add url with clear scheme - urls.append(url) - else: - logger.warning( - "Url '%s' is ignored. It is either a non-existing " - "path or lacks a specific scheme.", url, - ) - - return files, urls - - -class CollectedLinks(object): - - """ - Encapsulates all the Link objects collected by a call to - LinkCollector.collect_links(), stored separately as-- - - (1) links from the configured file locations, - (2) links from the configured find_links, and - (3) a dict mapping HTML page url to links from that page. - """ - - def __init__( - self, - files, # type: List[Link] - find_links, # type: List[Link] - pages, # type: Dict[str, List[Link]] - ): - # type: (...) -> None - """ - :param files: Links from file locations. - :param find_links: Links from find_links. - :param pages: A dict mapping HTML page url to links from that page. - """ - self.files = files - self.find_links = find_links - self.pages = pages - - -class LinkCollector(object): - - """ - Responsible for collecting Link objects from all configured locations, - making network requests as needed. - - The class's main method is its collect_links() method. - """ - - def __init__( - self, - session, # type: PipSession - search_scope, # type: SearchScope - ): - # type: (...) -> None - self.search_scope = search_scope - self.session = session - - @property - def find_links(self): - # type: () -> List[str] - return self.search_scope.find_links - - def _get_pages(self, locations, project_name): - # type: (Iterable[Link], str) -> Iterable[HTMLPage] - """ - Yields (page, page_url) from the given locations, skipping - locations that have errors. - """ - seen = set() # type: Set[Link] - for location in locations: - if location in seen: - continue - seen.add(location) - - page = _get_html_page(location, session=self.session) - if page is None: - continue - - yield page - - def collect_links(self, project_name): - # type: (str) -> CollectedLinks - """Find all available links for the given project name. - - :return: All the Link objects (unfiltered), as a CollectedLinks object. - """ - search_scope = self.search_scope - index_locations = search_scope.get_index_urls_locations(project_name) - index_file_loc, index_url_loc = group_locations(index_locations) - fl_file_loc, fl_url_loc = group_locations( - self.find_links, expand_dir=True, - ) - - file_links = [ - Link(url) for url in itertools.chain(index_file_loc, fl_file_loc) - ] - - # We trust every directly linked archive in find_links - find_link_links = [Link(url, '-f') for url in self.find_links] - - # We trust every url that the user has given us whether it was given - # via --index-url or --find-links. - # We want to filter out anything that does not have a secure origin. - url_locations = [ - link for link in itertools.chain( - (Link(url) for url in index_url_loc), - (Link(url) for url in fl_url_loc), - ) - if self.session.is_secure_origin(link) - ] - - logger.debug('%d location(s) to search for versions of %s:', - len(url_locations), project_name) - - for location in url_locations: - logger.debug('* %s', location) - - pages_links = {} - for page in self._get_pages(url_locations, project_name): - pages_links[page.url] = list(page.iter_links()) - - return CollectedLinks( - files=file_links, - find_links=find_link_links, - pages=pages_links, - ) - - def _check_link_requires_python( link, # type: Link version_info, # type: Tuple[int, int, int] @@ -1011,9 +637,8 @@ def __init__( @classmethod def create( cls, - search_scope, # type: SearchScope + link_collector, # type: LinkCollector selection_prefs, # type: SelectionPreferences - session=None, # type: Optional[PipSession] target_python=None, # type: Optional[TargetPython] ): # type: (...) -> PackageFinder @@ -1021,16 +646,10 @@ def create( :param selection_prefs: The candidate selection preferences, as a SelectionPreferences object. - :param session: The Session to use to make requests. :param target_python: The target Python interpreter to use when checking compatibility. If None (the default), a TargetPython object will be constructed from the running Python. """ - if session is None: - raise TypeError( - "PackageFinder.create() missing 1 required keyword argument: " - "'session'" - ) if target_python is None: target_python = TargetPython() @@ -1039,11 +658,6 @@ def create( allow_all_prereleases=selection_prefs.allow_all_prereleases, ) - link_collector = LinkCollector( - session=session, - search_scope=search_scope, - ) - return cls( candidate_prefs=candidate_prefs, link_collector=link_collector, @@ -1375,122 +989,3 @@ def _extract_version_from_fragment(fragment, canonical_name): if not version: return None return version - - -def _determine_base_url(document, page_url): - """Determine the HTML document's base URL. - - This looks for a ```` tag in the HTML document. If present, its href - attribute denotes the base URL of anchor tags in the document. If there is - no such tag (or if it does not have a valid href attribute), the HTML - file's URL is used as the base URL. - - :param document: An HTML document representation. The current - implementation expects the result of ``html5lib.parse()``. - :param page_url: The URL of the HTML document. - """ - for base in document.findall(".//base"): - href = base.get("href") - if href is not None: - return href - return page_url - - -def _get_encoding_from_headers(headers): - """Determine if we have any encoding information in our headers. - """ - if headers and "Content-Type" in headers: - content_type, params = cgi.parse_header(headers["Content-Type"]) - if "charset" in params: - return params['charset'] - return None - - -def _clean_link(url): - # type: (str) -> str - """Makes sure a link is fully encoded. That is, if a ' ' shows up in - the link, it will be rewritten to %20 (while not over-quoting - % or other characters).""" - # Split the URL into parts according to the general structure - # `scheme://netloc/path;parameters?query#fragment`. Note that the - # `netloc` can be empty and the URI will then refer to a local - # filesystem path. - result = urllib_parse.urlparse(url) - # In both cases below we unquote prior to quoting to make sure - # nothing is double quoted. - if result.netloc == "": - # On Windows the path part might contain a drive letter which - # should not be quoted. On Linux where drive letters do not - # exist, the colon should be quoted. We rely on urllib.request - # to do the right thing here. - path = urllib_request.pathname2url( - urllib_request.url2pathname(result.path)) - else: - # In addition to the `/` character we protect `@` so that - # revision strings in VCS URLs are properly parsed. - path = urllib_parse.quote(urllib_parse.unquote(result.path), safe="/@") - return urllib_parse.urlunparse(result._replace(path=path)) - - -def _create_link_from_element( - anchor, # type: HTMLElement - page_url, # type: str - base_url, # type: str -): - # type: (...) -> Optional[Link] - """ - Convert an anchor element in a simple repository page to a Link. - """ - href = anchor.get("href") - if not href: - return None - - url = _clean_link(urllib_parse.urljoin(base_url, href)) - pyrequire = anchor.get('data-requires-python') - pyrequire = unescape(pyrequire) if pyrequire else None - - yanked_reason = anchor.get('data-yanked') - if yanked_reason: - # This is a unicode string in Python 2 (and 3). - yanked_reason = unescape(yanked_reason) - - link = Link( - url, - comes_from=page_url, - requires_python=pyrequire, - yanked_reason=yanked_reason, - ) - - return link - - -class HTMLPage(object): - """Represents one page, along with its URL""" - - def __init__(self, content, url, headers=None): - # type: (bytes, str, MutableMapping[str, str]) -> None - self.content = content - self.url = url - self.headers = headers - - def __str__(self): - return redact_auth_from_url(self.url) - - def iter_links(self): - # type: () -> Iterable[Link] - """Yields all links in the page""" - document = html5lib.parse( - self.content, - transport_encoding=_get_encoding_from_headers(self.headers), - namespaceHTMLElements=False, - ) - base_url = _determine_base_url(document, self.url) - for anchor in document.findall(".//a"): - link = _create_link_from_element( - anchor, - page_url=self.url, - base_url=base_url, - ) - if link is None: - continue - yield link diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index 2a3c605e583..b585c93c395 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -3,8 +3,8 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._internal.utils.filetypes import WHEEL_EXTENSION from pip._internal.utils.misc import ( - WHEEL_EXTENSION, path_to_url, redact_auth_from_url, split_auth_from_netloc, @@ -15,7 +15,7 @@ if MYPY_CHECK_RUNNING: from typing import Optional, Text, Tuple, Union - from pip._internal.index import HTMLPage + from pip._internal.collector import HTMLPage from pip._internal.utils.hashes import Hashes diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 20e0301bed7..9bcab9b9935 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -13,12 +13,7 @@ make_distribution_for_install_requirement, ) from pip._internal.distributions.installed import InstalledDistribution -from pip._internal.download import ( - is_dir_url, - is_file_url, - unpack_url, - url_to_path, -) +from pip._internal.download import is_dir_url, is_file_url, unpack_url from pip._internal.exceptions import ( DirectoryUrlHashUnsupported, HashUnpinned, @@ -32,6 +27,7 @@ from pip._internal.utils.marker_files import write_delete_marker_file from pip._internal.utils.misc import display_path, normalize_path from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import url_to_path if MYPY_CHECK_RUNNING: from typing import Optional diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 28cef933221..702f4392f96 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -20,15 +20,16 @@ from pip._vendor.packaging.specifiers import Specifier from pip._vendor.pkg_resources import RequirementParseError, parse_requirements -from pip._internal.download import is_archive_file, is_url, url_to_path from pip._internal.exceptions import InstallationError from pip._internal.models.index import PyPI, TestPyPI from pip._internal.models.link import Link from pip._internal.pyproject import make_pyproject_path from pip._internal.req.req_install import InstallRequirement -from pip._internal.utils.misc import is_installable_dir, path_to_url +from pip._internal.utils.filetypes import ARCHIVE_EXTENSIONS +from pip._internal.utils.misc import is_installable_dir, path_to_url, splitext from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.vcs import vcs +from pip._internal.utils.urls import url_to_path +from pip._internal.vcs import is_url, vcs from pip._internal.wheel import Wheel if MYPY_CHECK_RUNNING: @@ -47,6 +48,15 @@ operators = Specifier._operators.keys() +def is_archive_file(name): + # type: (str) -> bool + """Return True if `name` is a considered as an archive file.""" + ext = splitext(name)[1].lower() + if ext in ARCHIVE_EXTENSIONS: + return True + return False + + def _strip_extras(path): # type: (str) -> Tuple[str, Optional[str]] m = re.match(r'^(.+)(\[[^\]]+\])$', path) @@ -60,6 +70,14 @@ def _strip_extras(path): return path_no_extras, extras +def convert_extras(extras): + # type: (Optional[str]) -> Set[str] + if extras: + return Requirement("placeholder" + extras.lower()).extras + else: + return set() + + def parse_editable(editable_req): # type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]] """Parses an editable requirement into: @@ -201,6 +219,60 @@ def install_req_from_editable( ) +def _looks_like_path(name): + # type: (str) -> bool + """Checks whether the string "looks like" a path on the filesystem. + + This does not check whether the target actually exists, only judge from the + appearance. + + Returns true if any of the following conditions is true: + * a path separator is found (either os.path.sep or os.path.altsep); + * a dot is found (which represents the current directory). + """ + if os.path.sep in name: + return True + if os.path.altsep is not None and os.path.altsep in name: + return True + if name.startswith("."): + return True + return False + + +def _get_url_from_path(path, name): + # type: (str, str) -> str + """ + First, it checks whether a provided path is an installable directory + (e.g. it has a setup.py). If it is, returns the path. + + If false, check if the path is an archive file (such as a .whl). + The function checks if the path is a file. If false, if the path has + an @, it will treat it as a PEP 440 URL requirement and return the path. + """ + if _looks_like_path(name) and os.path.isdir(path): + if is_installable_dir(path): + return path_to_url(path) + raise InstallationError( + "Directory %r is not installable. Neither 'setup.py' " + "nor 'pyproject.toml' found." % name + ) + if not is_archive_file(path): + return None + if os.path.isfile(path): + return path_to_url(path) + urlreq_parts = name.split('@', 1) + if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]): + # If the path contains '@' and the part before it does not look + # like a path, try to treat it as a PEP 440 URL req instead. + return None + logger.warning( + 'Requirement %r looks like a filename, but the ' + 'file does not exist', + name + ) + return path_to_url(path) + + def install_req_from_line( name, # type: str comes_from=None, # type: Optional[Union[str, InstallRequirement]] @@ -241,26 +313,9 @@ def install_req_from_line( link = Link(name) else: p, extras_as_string = _strip_extras(path) - looks_like_dir = os.path.isdir(p) and ( - os.path.sep in name or - (os.path.altsep is not None and os.path.altsep in name) or - name.startswith('.') - ) - if looks_like_dir: - if not is_installable_dir(p): - raise InstallationError( - "Directory %r is not installable. Neither 'setup.py' " - "nor 'pyproject.toml' found." % name - ) - link = Link(path_to_url(p)) - elif is_archive_file(p): - if not os.path.isfile(p): - logger.warning( - 'Requirement %r looks like a filename, but the ' - 'file does not exist', - name - ) - link = Link(path_to_url(p)) + url = _get_url_from_path(p, name) + if url is not None: + link = Link(url) # it's a local file, dir, or url if link: @@ -281,10 +336,13 @@ def install_req_from_line( else: req_as_string = name - if extras_as_string: - extras = Requirement("placeholder" + extras_as_string.lower()).extras - else: - extras = () + extras = convert_extras(extras_as_string) + + def with_source(text): + if not line_source: + return text + return '{} (from {})'.format(text, line_source) + if req_as_string is not None: try: req = Requirement(req_as_string) @@ -297,12 +355,8 @@ def install_req_from_line( add_msg = "= is not a valid operator. Did you mean == ?" else: add_msg = '' - if line_source is None: - source = '' - else: - source = ' (from {})'.format(line_source) - msg = ( - 'Invalid requirement: {!r}{}'.format(req_as_string, source) + msg = with_source( + 'Invalid requirement: {!r}'.format(req_as_string) ) if add_msg: msg += '\nHint: {}'.format(add_msg) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 379a8baa51d..cab26206671 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -45,8 +45,8 @@ hide_url, redact_auth_from_url, rmtree, - unpack_file, ) +from pip._internal.utils.unpacking import unpack_file from pip._internal.utils.packaging import get_metadata from pip._internal.utils.setuptools_build import make_setuptools_shim_args from pip._internal.utils.temp_dir import TempDirectory @@ -323,7 +323,7 @@ def from_path(self): s += '->' + comes_from return s - def build_location(self, build_dir): + def ensure_build_location(self, build_dir): # type: (str) -> str assert build_dir is not None if self._temp_build_dir.path is not None: @@ -371,7 +371,7 @@ def _correct_build_location(self): old_location = self._temp_build_dir.path self._temp_build_dir.path = None - new_location = self.build_location(self._ideal_build_dir) + new_location = self.ensure_build_location(self._ideal_build_dir) if os.path.exists(new_location): raise InstallationError( 'A package already exists in %s; please remove it to continue' @@ -764,7 +764,7 @@ def ensure_has_source_dir(self, parent_dir): :return: self.source_dir """ if self.source_dir is None: - self.source_dir = self.build_location(parent_dir) + self.source_dir = self.ensure_build_location(parent_dir) # For editable installations def install_editable( diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index e35e47615cd..6b551c91bec 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -267,14 +267,16 @@ def _get_file_stash(self, path): def stash(self, path): # type: (str) -> str """Stashes the directory or file and returns its new location. + Handle symlinks as files to avoid modifying the symlink targets. """ - if os.path.isdir(path): + path_is_dir = os.path.isdir(path) and not os.path.islink(path) + if path_is_dir: new_path = self._get_directory_stash(path) else: new_path = self._get_file_stash(path) self._moves.append((path, new_path)) - if os.path.isdir(path) and os.path.isdir(new_path): + if (path_is_dir and os.path.isdir(new_path)): # If we're moving a directory, we need to # remove the destination first or else it will be # moved to inside the existing directory. @@ -301,7 +303,7 @@ def rollback(self): for new_path, path in self._moves: try: logger.debug('Replacing %s from %s', new_path, path) - if os.path.isfile(new_path): + if os.path.isfile(new_path) or os.path.islink(new_path): os.unlink(new_path) elif os.path.isdir(new_path): rmtree(new_path) diff --git a/src/pip/_internal/utils/filesystem.py b/src/pip/_internal/utils/filesystem.py index c5233ebbc71..f4a389cd92f 100644 --- a/src/pip/_internal/utils/filesystem.py +++ b/src/pip/_internal/utils/filesystem.py @@ -2,8 +2,26 @@ import os.path import shutil import stat +from contextlib import contextmanager +from tempfile import NamedTemporaryFile + +# NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is +# why we ignore the type on this import. +from pip._vendor.retrying import retry # type: ignore +from pip._vendor.six import PY2 from pip._internal.utils.compat import get_path_uid +from pip._internal.utils.misc import cast +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import BinaryIO, Iterator + + class NamedTemporaryFileResult(BinaryIO): + @property + def file(self): + # type: () -> BinaryIO + pass def check_path_owner(path): @@ -59,3 +77,39 @@ def copy2_fixed(src, dest): def is_socket(path): # type: (str) -> bool return stat.S_ISSOCK(os.lstat(path).st_mode) + + +@contextmanager +def adjacent_tmp_file(path): + # type: (str) -> Iterator[NamedTemporaryFileResult] + """Given a path to a file, open a temp file next to it securely and ensure + it is written to disk after the context reaches its end. + """ + with NamedTemporaryFile( + delete=False, + dir=os.path.dirname(path), + prefix=os.path.basename(path), + suffix='.tmp', + ) as f: + result = cast('NamedTemporaryFileResult', f) + try: + yield result + finally: + result.file.flush() + os.fsync(result.file.fileno()) + + +_replace_retry = retry(stop_max_delay=1000, wait_fixed=250) + +if PY2: + @_replace_retry + def replace(src, dest): + # type: (str, str) -> None + try: + os.rename(src, dest) + except OSError: + os.remove(dest) + os.rename(src, dest) + +else: + replace = _replace_retry(os.replace) diff --git a/src/pip/_internal/utils/filetypes.py b/src/pip/_internal/utils/filetypes.py new file mode 100644 index 00000000000..251e3e933d3 --- /dev/null +++ b/src/pip/_internal/utils/filetypes.py @@ -0,0 +1,11 @@ +"""Filetype information. +""" + +WHEEL_EXTENSION = '.whl' +BZ2_EXTENSIONS = ('.tar.bz2', '.tbz') +XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', '.tar.lz', '.tar.lzma') +ZIP_EXTENSIONS = ('.zip', WHEEL_EXTENSION) +TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar') +ARCHIVE_EXTENSIONS = ( + ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS +) diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 29cd0541d00..5f13f975c29 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -10,13 +10,10 @@ import logging import os import posixpath -import re import shutil import stat import subprocess import sys -import tarfile -import zipfile from collections import deque from pip._vendor import pkg_resources @@ -57,11 +54,10 @@ if MYPY_CHECK_RUNNING: from typing import ( - Any, AnyStr, Container, Iterable, List, Mapping, Match, Optional, Text, + Any, AnyStr, Container, Iterable, List, Mapping, Optional, Text, Tuple, Union, cast, ) from pip._vendor.pkg_resources import Distribution - from pip._internal.models.link import Link from pip._internal.utils.ui import SpinnerInterface VersionInfo = Tuple[int, int, int] @@ -76,13 +72,10 @@ def cast(type_, value): # type: ignore __all__ = ['rmtree', 'display_path', 'backup_dir', 'ask', 'splitext', 'format_size', 'is_installable_dir', - 'is_svn_page', 'file_contents', - 'split_leading_dir', 'has_leading_dir', 'normalize_path', 'renames', 'get_prog', - 'unzip_file', 'untar_file', 'unpack_file', 'call_subprocess', + 'call_subprocess', 'captured_stdout', 'ensure_dir', - 'ARCHIVE_EXTENSIONS', 'SUPPORTED_EXTENSIONS', 'WHEEL_EXTENSION', 'get_installed_version', 'remove_auth_from_url'] @@ -91,28 +84,6 @@ def cast(type_, value): # type: ignore LOG_DIVIDER = '----------------------------------------' -WHEEL_EXTENSION = '.whl' -BZ2_EXTENSIONS = ('.tar.bz2', '.tbz') -XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', '.tar.lz', '.tar.lzma') -ZIP_EXTENSIONS = ('.zip', WHEEL_EXTENSION) -TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar') -ARCHIVE_EXTENSIONS = ( - ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS) -SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS - -try: - import bz2 # noqa - SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS -except ImportError: - logger.debug('bz2 module is not available') - -try: - # Only for Python 3.3+ - import lzma # noqa - SUPPORTED_EXTENSIONS += XZ_EXTENSIONS -except ImportError: - logger.debug('lzma module is not available') - def get_pip_version(): # type: () -> str @@ -181,8 +152,13 @@ def rmtree_errorhandler(func, path, exc_info): """On Windows, the files in .svn are read-only, so when rmtree() tries to remove them, an exception is thrown. We catch that here, remove the read-only attribute, and hopefully continue without problems.""" - # if file type currently read only - if os.stat(path).st_mode & stat.S_IREAD: + try: + has_attr_readonly = not (os.stat(path).st_mode & stat.S_IWRITE) + except (IOError, OSError): + # it's equivalent to os.path.exists + return + + if has_attr_readonly: # convert to read/write os.chmod(path, stat.S_IWRITE) # use the original function to repeat the operation @@ -326,21 +302,6 @@ def is_installable_dir(path): return False -def is_svn_page(html): - # type: (Union[str, Text]) -> Optional[Match[Union[str, Text]]] - """ - Returns true if the page appears to be the index page of an svn repository - """ - return (re.search(r'[^<]*Revision \d+:', html) and - re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I)) - - -def file_contents(filename): - # type: (str) -> Text - with open(filename, 'rb') as fp: - return fp.read().decode('utf-8') - - def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): """Yield pieces of data from a file-like object until EOF.""" while True: @@ -350,34 +311,6 @@ def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): yield chunk -def split_leading_dir(path): - # type: (Union[str, Text]) -> List[Union[str, Text]] - path = path.lstrip('/').lstrip('\\') - if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) or - '\\' not in path): - return path.split('/', 1) - elif '\\' in path: - return path.split('\\', 1) - else: - return [path, ''] - - -def has_leading_dir(paths): - # type: (Iterable[Union[str, Text]]) -> bool - """Returns true if all the paths have the same leading path name - (i.e., everything is in one subdirectory in an archive)""" - common_prefix = None - for path in paths: - prefix, rest = split_leading_dir(path) - if not prefix: - return False - elif common_prefix is None: - common_prefix = prefix - elif prefix != common_prefix: - return False - return True - - def normalize_path(path, resolve_symlinks=True): # type: (str, bool) -> str """ @@ -427,10 +360,12 @@ def is_local(path): If we're not in a virtualenv, all paths are considered "local." + Caution: this function assumes the head of path has been normalized + with normalize_path. """ if not running_under_virtualenv(): return True - return normalize_path(path).startswith(normalize_path(sys.prefix)) + return path.startswith(normalize_path(sys.prefix)) def dist_is_local(dist): @@ -450,8 +385,7 @@ def dist_in_usersite(dist): """ Return True if given Distribution is installed in user site. """ - norm_path = normalize_path(dist_location(dist)) - return norm_path.startswith(normalize_path(user_site)) + return dist_location(dist).startswith(normalize_path(user_site)) def dist_in_site_packages(dist): @@ -460,9 +394,7 @@ def dist_in_site_packages(dist): Return True if given Distribution is installed in sysconfig.get_python_lib(). """ - return normalize_path( - dist_location(dist) - ).startswith(normalize_path(site_packages)) + return dist_location(dist).startswith(normalize_path(site_packages)) def dist_is_editable(dist): @@ -593,180 +525,12 @@ def dist_location(dist): packages, where dist.location is the source code location, and we want to know where the egg-link file is. + The returned location is normalized (in particular, with symlinks removed). """ egg_link = egg_link_path(dist) if egg_link: - return egg_link - return dist.location - - -def current_umask(): - """Get the current umask which involves having to set it temporarily.""" - mask = os.umask(0) - os.umask(mask) - return mask - - -def unzip_file(filename, location, flatten=True): - # type: (str, str, bool) -> None - """ - Unzip the file (with path `filename`) to the destination `location`. All - files are written based on system defaults and umask (i.e. permissions are - not preserved), except that regular file members with any execute - permissions (user, group, or world) have "chmod +x" applied after being - written. Note that for windows, any execute changes using os.chmod are - no-ops per the python docs. - """ - ensure_dir(location) - zipfp = open(filename, 'rb') - try: - zip = zipfile.ZipFile(zipfp, allowZip64=True) - leading = has_leading_dir(zip.namelist()) and flatten - for info in zip.infolist(): - name = info.filename - fn = name - if leading: - fn = split_leading_dir(name)[1] - fn = os.path.join(location, fn) - dir = os.path.dirname(fn) - if fn.endswith('/') or fn.endswith('\\'): - # A directory - ensure_dir(fn) - else: - ensure_dir(dir) - # Don't use read() to avoid allocating an arbitrarily large - # chunk of memory for the file's content - fp = zip.open(name) - try: - with open(fn, 'wb') as destfp: - shutil.copyfileobj(fp, destfp) - finally: - fp.close() - mode = info.external_attr >> 16 - # if mode and regular file and any execute permissions for - # user/group/world? - if mode and stat.S_ISREG(mode) and mode & 0o111: - # make dest file have execute for user/group/world - # (chmod +x) no-op on windows per python docs - os.chmod(fn, (0o777 - current_umask() | 0o111)) - finally: - zipfp.close() - - -def untar_file(filename, location): - # type: (str, str) -> None - """ - Untar the file (with path `filename`) to the destination `location`. - All files are written based on system defaults and umask (i.e. permissions - are not preserved), except that regular file members with any execute - permissions (user, group, or world) have "chmod +x" applied after being - written. Note that for windows, any execute changes using os.chmod are - no-ops per the python docs. - """ - ensure_dir(location) - if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): - mode = 'r:gz' - elif filename.lower().endswith(BZ2_EXTENSIONS): - mode = 'r:bz2' - elif filename.lower().endswith(XZ_EXTENSIONS): - mode = 'r:xz' - elif filename.lower().endswith('.tar'): - mode = 'r' - else: - logger.warning( - 'Cannot determine compression type for file %s', filename, - ) - mode = 'r:*' - tar = tarfile.open(filename, mode) - try: - leading = has_leading_dir([ - member.name for member in tar.getmembers() - ]) - for member in tar.getmembers(): - fn = member.name - if leading: - # https://github.com/python/mypy/issues/1174 - fn = split_leading_dir(fn)[1] # type: ignore - path = os.path.join(location, fn) - if member.isdir(): - ensure_dir(path) - elif member.issym(): - try: - # https://github.com/python/typeshed/issues/2673 - tar._extract_member(member, path) # type: ignore - except Exception as exc: - # Some corrupt tar files seem to produce this - # (specifically bad symlinks) - logger.warning( - 'In the tar file %s the member %s is invalid: %s', - filename, member.name, exc, - ) - continue - else: - try: - fp = tar.extractfile(member) - except (KeyError, AttributeError) as exc: - # Some corrupt tar files seem to produce this - # (specifically bad symlinks) - logger.warning( - 'In the tar file %s the member %s is invalid: %s', - filename, member.name, exc, - ) - continue - ensure_dir(os.path.dirname(path)) - with open(path, 'wb') as destfp: - shutil.copyfileobj(fp, destfp) - fp.close() - # Update the timestamp (useful for cython compiled files) - # https://github.com/python/typeshed/issues/2673 - tar.utime(member, path) # type: ignore - # member have any execute permissions for user/group/world? - if member.mode & 0o111: - # make dest file have execute for user/group/world - # no-op on windows per python docs - os.chmod(path, (0o777 - current_umask() | 0o111)) - finally: - tar.close() - - -def unpack_file( - filename, # type: str - location, # type: str - content_type, # type: Optional[str] - link # type: Optional[Link] -): - # type: (...) -> None - filename = os.path.realpath(filename) - if (content_type == 'application/zip' or - filename.lower().endswith(ZIP_EXTENSIONS) or - zipfile.is_zipfile(filename)): - unzip_file( - filename, - location, - flatten=not filename.endswith('.whl') - ) - elif (content_type == 'application/x-gzip' or - tarfile.is_tarfile(filename) or - filename.lower().endswith( - TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS)): - untar_file(filename, location) - elif (content_type and content_type.startswith('text/html') and - is_svn_page(file_contents(filename))): - # We don't really care about this - from pip._internal.vcs.subversion import Subversion - hidden_url = hide_url('svn+' + link.url) - Subversion().unpack(location, url=hidden_url) - else: - # FIXME: handle? - # FIXME: magic signatures? - logger.critical( - 'Cannot unpack file %s (downloaded from %s, content-type: %s); ' - 'cannot detect archive format', - filename, location, content_type, - ) - raise InstallationError( - 'Cannot determine archive format of %s' % location - ) + return normalize_path(egg_link) + return normalize_path(dist.location) def make_command(*args): diff --git a/src/pip/_internal/utils/outdated.py b/src/pip/_internal/utils/outdated.py index e0c90e15c49..ca219c618a9 100644 --- a/src/pip/_internal/utils/outdated.py +++ b/src/pip/_internal/utils/outdated.py @@ -7,21 +7,31 @@ import os.path import sys -from pip._vendor import lockfile, pkg_resources +from pip._vendor import pkg_resources from pip._vendor.packaging import version as packaging_version from pip._vendor.six import ensure_binary -from pip._internal.cli.cmdoptions import make_search_scope +from pip._internal.collector import LinkCollector from pip._internal.index import PackageFinder +from pip._internal.models.search_scope import SearchScope from pip._internal.models.selection_prefs import SelectionPreferences from pip._internal.utils.compat import WINDOWS -from pip._internal.utils.filesystem import check_path_owner -from pip._internal.utils.misc import ensure_dir, get_installed_version +from pip._internal.utils.filesystem import ( + adjacent_tmp_file, + check_path_owner, + replace, +) +from pip._internal.utils.misc import ( + ensure_dir, + get_installed_version, + redact_auth_from_url, +) from pip._internal.utils.packaging import get_installer from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: import optparse + from optparse import Values from typing import Any, Dict, Text, Union from pip._internal.download import PipSession @@ -32,6 +42,37 @@ logger = logging.getLogger(__name__) +def make_link_collector( + session, # type: PipSession + options, # type: Values + suppress_no_index=False, # type: bool +): + # type: (...) -> LinkCollector + """ + :param session: The Session to use to make requests. + :param suppress_no_index: Whether to ignore the --no-index option + when constructing the SearchScope object. + """ + index_urls = [options.index_url] + options.extra_index_urls + if options.no_index and not suppress_no_index: + logger.debug( + 'Ignoring indexes: %s', + ','.join(redact_auth_from_url(url) for url in index_urls), + ) + index_urls = [] + + # Make sure find_links is a list before passing to create(). + find_links = options.find_links or [] + + search_scope = SearchScope.create( + find_links=find_links, index_urls=index_urls, + ) + + link_collector = LinkCollector(session=session, search_scope=search_scope) + + return link_collector + + def _get_statefile_name(key): # type: (Union[str, Text]) -> str key_bytes = ensure_binary(key) @@ -86,12 +127,16 @@ def save(self, pypi_version, current_time): text = json.dumps(state, sort_keys=True, separators=(",", ":")) - # Attempt to write out our version check file - with lockfile.LockFile(self.statefile_path): + with adjacent_tmp_file(self.statefile_path) as f: + f.write(ensure_binary(text)) + + try: # Since we have a prefix-specific state file, we can just # overwrite whatever is there, no need to check. - with open(self.statefile_path, "w") as statefile: - statefile.write(text) + replace(f.name, self.statefile_path) + except OSError: + # Best effort. + pass def was_installed_by_pip(pkg): @@ -139,7 +184,11 @@ def pip_version_check(session, options): # Refresh the version if we need to or just see if we need to warn if pypi_version is None: # Lets use PackageFinder to see what the latest pip version is - search_scope = make_search_scope(options, suppress_no_index=True) + link_collector = make_link_collector( + session, + options=options, + suppress_no_index=True, + ) # Pass allow_yanked=False so we don't suggest upgrading to a # yanked version. @@ -149,9 +198,8 @@ def pip_version_check(session, options): ) finder = PackageFinder.create( - search_scope=search_scope, + link_collector=link_collector, selection_prefs=selection_prefs, - session=session, ) best_candidate = finder.find_best_candidate("pip").best_candidate if best_candidate is None: @@ -180,7 +228,7 @@ def pip_version_check(session, options): else: pip_cmd = "pip" logger.warning( - "You are using pip version %s, however version %s is " + "You are using pip version %s; however, version %s is " "available.\nYou should consider upgrading via the " "'%s install --upgrade pip' command.", pip_version, pypi_version, pip_cmd diff --git a/src/pip/_internal/utils/unpacking.py b/src/pip/_internal/utils/unpacking.py new file mode 100644 index 00000000000..92424da5ad3 --- /dev/null +++ b/src/pip/_internal/utils/unpacking.py @@ -0,0 +1,245 @@ +"""Utilities related archives. +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import logging +import os +import shutil +import stat +import tarfile +import zipfile + +from pip._internal.exceptions import InstallationError +from pip._internal.utils.filetypes import ( + BZ2_EXTENSIONS, + TAR_EXTENSIONS, + XZ_EXTENSIONS, + ZIP_EXTENSIONS, +) +from pip._internal.utils.misc import ensure_dir +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Iterable, List, Optional, Text, Union + + +logger = logging.getLogger(__name__) + + +SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS + +try: + import bz2 # noqa + SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS +except ImportError: + logger.debug('bz2 module is not available') + +try: + # Only for Python 3.3+ + import lzma # noqa + SUPPORTED_EXTENSIONS += XZ_EXTENSIONS +except ImportError: + logger.debug('lzma module is not available') + + +def current_umask(): + """Get the current umask which involves having to set it temporarily.""" + mask = os.umask(0) + os.umask(mask) + return mask + + +def split_leading_dir(path): + # type: (Union[str, Text]) -> List[Union[str, Text]] + path = path.lstrip('/').lstrip('\\') + if ( + '/' in path and ( + ('\\' in path and path.find('/') < path.find('\\')) or + '\\' not in path + ) + ): + return path.split('/', 1) + elif '\\' in path: + return path.split('\\', 1) + else: + return [path, ''] + + +def has_leading_dir(paths): + # type: (Iterable[Union[str, Text]]) -> bool + """Returns true if all the paths have the same leading path name + (i.e., everything is in one subdirectory in an archive)""" + common_prefix = None + for path in paths: + prefix, rest = split_leading_dir(path) + if not prefix: + return False + elif common_prefix is None: + common_prefix = prefix + elif prefix != common_prefix: + return False + return True + + +def unzip_file(filename, location, flatten=True): + # type: (str, str, bool) -> None + """ + Unzip the file (with path `filename`) to the destination `location`. All + files are written based on system defaults and umask (i.e. permissions are + not preserved), except that regular file members with any execute + permissions (user, group, or world) have "chmod +x" applied after being + written. Note that for windows, any execute changes using os.chmod are + no-ops per the python docs. + """ + ensure_dir(location) + zipfp = open(filename, 'rb') + try: + zip = zipfile.ZipFile(zipfp, allowZip64=True) + leading = has_leading_dir(zip.namelist()) and flatten + for info in zip.infolist(): + name = info.filename + fn = name + if leading: + fn = split_leading_dir(name)[1] + fn = os.path.join(location, fn) + dir = os.path.dirname(fn) + if fn.endswith('/') or fn.endswith('\\'): + # A directory + ensure_dir(fn) + else: + ensure_dir(dir) + # Don't use read() to avoid allocating an arbitrarily large + # chunk of memory for the file's content + fp = zip.open(name) + try: + with open(fn, 'wb') as destfp: + shutil.copyfileobj(fp, destfp) + finally: + fp.close() + mode = info.external_attr >> 16 + # if mode and regular file and any execute permissions for + # user/group/world? + if mode and stat.S_ISREG(mode) and mode & 0o111: + # make dest file have execute for user/group/world + # (chmod +x) no-op on windows per python docs + os.chmod(fn, (0o777 - current_umask() | 0o111)) + finally: + zipfp.close() + + +def untar_file(filename, location): + # type: (str, str) -> None + """ + Untar the file (with path `filename`) to the destination `location`. + All files are written based on system defaults and umask (i.e. permissions + are not preserved), except that regular file members with any execute + permissions (user, group, or world) have "chmod +x" applied after being + written. Note that for windows, any execute changes using os.chmod are + no-ops per the python docs. + """ + ensure_dir(location) + if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): + mode = 'r:gz' + elif filename.lower().endswith(BZ2_EXTENSIONS): + mode = 'r:bz2' + elif filename.lower().endswith(XZ_EXTENSIONS): + mode = 'r:xz' + elif filename.lower().endswith('.tar'): + mode = 'r' + else: + logger.warning( + 'Cannot determine compression type for file %s', filename, + ) + mode = 'r:*' + tar = tarfile.open(filename, mode) + try: + leading = has_leading_dir([ + member.name for member in tar.getmembers() + ]) + for member in tar.getmembers(): + fn = member.name + if leading: + # https://github.com/python/mypy/issues/1174 + fn = split_leading_dir(fn)[1] # type: ignore + path = os.path.join(location, fn) + if member.isdir(): + ensure_dir(path) + elif member.issym(): + try: + # https://github.com/python/typeshed/issues/2673 + tar._extract_member(member, path) # type: ignore + except Exception as exc: + # Some corrupt tar files seem to produce this + # (specifically bad symlinks) + logger.warning( + 'In the tar file %s the member %s is invalid: %s', + filename, member.name, exc, + ) + continue + else: + try: + fp = tar.extractfile(member) + except (KeyError, AttributeError) as exc: + # Some corrupt tar files seem to produce this + # (specifically bad symlinks) + logger.warning( + 'In the tar file %s the member %s is invalid: %s', + filename, member.name, exc, + ) + continue + ensure_dir(os.path.dirname(path)) + with open(path, 'wb') as destfp: + shutil.copyfileobj(fp, destfp) + fp.close() + # Update the timestamp (useful for cython compiled files) + # https://github.com/python/typeshed/issues/2673 + tar.utime(member, path) # type: ignore + # member have any execute permissions for user/group/world? + if member.mode & 0o111: + # make dest file have execute for user/group/world + # no-op on windows per python docs + os.chmod(path, (0o777 - current_umask() | 0o111)) + finally: + tar.close() + + +def unpack_file( + filename, # type: str + location, # type: str + content_type, # type: Optional[str] +): + # type: (...) -> None + filename = os.path.realpath(filename) + if ( + content_type == 'application/zip' or + filename.lower().endswith(ZIP_EXTENSIONS) or + zipfile.is_zipfile(filename) + ): + unzip_file( + filename, + location, + flatten=not filename.endswith('.whl') + ) + elif ( + content_type == 'application/x-gzip' or + tarfile.is_tarfile(filename) or + filename.lower().endswith( + TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS + ) + ): + untar_file(filename, location) + else: + # FIXME: handle? + # FIXME: magic signatures? + logger.critical( + 'Cannot unpack file %s (downloaded from %s, content-type: %s); ' + 'cannot detect archive format', + filename, location, content_type, + ) + raise InstallationError( + 'Cannot determine archive format of {}'.format(location) + ) diff --git a/src/pip/_internal/utils/urls.py b/src/pip/_internal/utils/urls.py new file mode 100644 index 00000000000..9c5385044c7 --- /dev/null +++ b/src/pip/_internal/utils/urls.py @@ -0,0 +1,42 @@ +import sys + +from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.six.moves.urllib import request as urllib_request + +from pip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Text, Union + + +def get_url_scheme(url): + # type: (Union[str, Text]) -> Optional[Text] + if ':' not in url: + return None + return url.split(':', 1)[0].lower() + + +def url_to_path(url): + # type: (str) -> str + """ + Convert a file: URL to a path. + """ + assert url.startswith('file:'), ( + "You can only turn file: urls into filenames (not %r)" % url) + + _, netloc, path, _, _ = urllib_parse.urlsplit(url) + + if not netloc or netloc == 'localhost': + # According to RFC 8089, same as empty authority. + netloc = '' + elif sys.platform == 'win32': + # If we have a UNC path, prepend UNC share notation. + netloc = '\\\\' + netloc + else: + raise ValueError( + 'non-local file URIs are not supported on this platform: %r' + % url + ) + + path = urllib_request.url2pathname(netloc + path) + return path diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index cb573ab6dc2..75b5589c53d 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -3,7 +3,7 @@ # (The test directory and imports protected by MYPY_CHECK_RUNNING may # still need to import from a vcs sub-package.) from pip._internal.vcs.versioncontrol import ( # noqa: F401 - RemoteNotFoundError, make_vcs_requirement_url, vcs, + RemoteNotFoundError, is_url, make_vcs_requirement_url, vcs, ) # Import all vcs modules to register each VCS in the VcsSupport object. import pip._internal.vcs.bazaar diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 40740e97867..27610602f16 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -22,6 +22,7 @@ rmtree, ) from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import get_url_scheme if MYPY_CHECK_RUNNING: from typing import ( @@ -39,6 +40,17 @@ logger = logging.getLogger(__name__) +def is_url(name): + # type: (Union[str, Text]) -> bool + """ + Return true if the name looks like a URL. + """ + scheme = get_url_scheme(name) + if scheme is None: + return False + return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes + + def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): """ Return the URL for a VCS requirement. diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index 05747c8fe31..5ae85a6ec44 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -1146,7 +1146,7 @@ def build( req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. - req.source_dir = req.build_location( + req.source_dir = req.ensure_build_location( self.preparer.build_dir ) # Update the link for this. diff --git a/src/pip/_vendor/__init__.py b/src/pip/_vendor/__init__.py index 2f824556113..a0fcb8e2cc4 100644 --- a/src/pip/_vendor/__init__.py +++ b/src/pip/_vendor/__init__.py @@ -64,7 +64,6 @@ def vendored(modulename): vendored("distlib") vendored("distro") vendored("html5lib") - vendored("lockfile") vendored("six") vendored("six.moves") vendored("six.moves.urllib") diff --git a/src/pip/_vendor/lockfile.pyi b/src/pip/_vendor/lockfile.pyi deleted file mode 100644 index 6e577ca7d81..00000000000 --- a/src/pip/_vendor/lockfile.pyi +++ /dev/null @@ -1 +0,0 @@ -from lockfile import * \ No newline at end of file diff --git a/src/pip/_vendor/lockfile/LICENSE b/src/pip/_vendor/lockfile/LICENSE deleted file mode 100644 index 610c0793f71..00000000000 --- a/src/pip/_vendor/lockfile/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -This is the MIT license: http://www.opensource.org/licenses/mit-license.php - -Copyright (c) 2007 Skip Montanaro. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to -deal in the Software without restriction, including without limitation the -rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -sell copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. diff --git a/src/pip/_vendor/lockfile/__init__.py b/src/pip/_vendor/lockfile/__init__.py deleted file mode 100644 index a6f44a55c63..00000000000 --- a/src/pip/_vendor/lockfile/__init__.py +++ /dev/null @@ -1,347 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -lockfile.py - Platform-independent advisory file locks. - -Requires Python 2.5 unless you apply 2.4.diff -Locking is done on a per-thread basis instead of a per-process basis. - -Usage: - ->>> lock = LockFile('somefile') ->>> try: -... lock.acquire() -... except AlreadyLocked: -... print 'somefile', 'is locked already.' -... except LockFailed: -... print 'somefile', 'can\\'t be locked.' -... else: -... print 'got lock' -got lock ->>> print lock.is_locked() -True ->>> lock.release() - ->>> lock = LockFile('somefile') ->>> print lock.is_locked() -False ->>> with lock: -... print lock.is_locked() -True ->>> print lock.is_locked() -False - ->>> lock = LockFile('somefile') ->>> # It is okay to lock twice from the same thread... ->>> with lock: -... lock.acquire() -... ->>> # Though no counter is kept, so you can't unlock multiple times... ->>> print lock.is_locked() -False - -Exceptions: - - Error - base class for other exceptions - LockError - base class for all locking exceptions - AlreadyLocked - Another thread or process already holds the lock - LockFailed - Lock failed for some other reason - UnlockError - base class for all unlocking exceptions - AlreadyUnlocked - File was not locked. - NotMyLock - File was locked but not by the current thread/process -""" - -from __future__ import absolute_import - -import functools -import os -import socket -import threading -import warnings - -# Work with PEP8 and non-PEP8 versions of threading module. -if not hasattr(threading, "current_thread"): - threading.current_thread = threading.currentThread -if not hasattr(threading.Thread, "get_name"): - threading.Thread.get_name = threading.Thread.getName - -__all__ = ['Error', 'LockError', 'LockTimeout', 'AlreadyLocked', - 'LockFailed', 'UnlockError', 'NotLocked', 'NotMyLock', - 'LinkFileLock', 'MkdirFileLock', 'SQLiteFileLock', - 'LockBase', 'locked'] - - -class Error(Exception): - """ - Base class for other exceptions. - - >>> try: - ... raise Error - ... except Exception: - ... pass - """ - pass - - -class LockError(Error): - """ - Base class for error arising from attempts to acquire the lock. - - >>> try: - ... raise LockError - ... except Error: - ... pass - """ - pass - - -class LockTimeout(LockError): - """Raised when lock creation fails within a user-defined period of time. - - >>> try: - ... raise LockTimeout - ... except LockError: - ... pass - """ - pass - - -class AlreadyLocked(LockError): - """Some other thread/process is locking the file. - - >>> try: - ... raise AlreadyLocked - ... except LockError: - ... pass - """ - pass - - -class LockFailed(LockError): - """Lock file creation failed for some other reason. - - >>> try: - ... raise LockFailed - ... except LockError: - ... pass - """ - pass - - -class UnlockError(Error): - """ - Base class for errors arising from attempts to release the lock. - - >>> try: - ... raise UnlockError - ... except Error: - ... pass - """ - pass - - -class NotLocked(UnlockError): - """Raised when an attempt is made to unlock an unlocked file. - - >>> try: - ... raise NotLocked - ... except UnlockError: - ... pass - """ - pass - - -class NotMyLock(UnlockError): - """Raised when an attempt is made to unlock a file someone else locked. - - >>> try: - ... raise NotMyLock - ... except UnlockError: - ... pass - """ - pass - - -class _SharedBase(object): - def __init__(self, path): - self.path = path - - def acquire(self, timeout=None): - """ - Acquire the lock. - - * If timeout is omitted (or None), wait forever trying to lock the - file. - - * If timeout > 0, try to acquire the lock for that many seconds. If - the lock period expires and the file is still locked, raise - LockTimeout. - - * If timeout <= 0, raise AlreadyLocked immediately if the file is - already locked. - """ - raise NotImplemented("implement in subclass") - - def release(self): - """ - Release the lock. - - If the file is not locked, raise NotLocked. - """ - raise NotImplemented("implement in subclass") - - def __enter__(self): - """ - Context manager support. - """ - self.acquire() - return self - - def __exit__(self, *_exc): - """ - Context manager support. - """ - self.release() - - def __repr__(self): - return "<%s: %r>" % (self.__class__.__name__, self.path) - - -class LockBase(_SharedBase): - """Base class for platform-specific lock classes.""" - def __init__(self, path, threaded=True, timeout=None): - """ - >>> lock = LockBase('somefile') - >>> lock = LockBase('somefile', threaded=False) - """ - super(LockBase, self).__init__(path) - self.lock_file = os.path.abspath(path) + ".lock" - self.hostname = socket.gethostname() - self.pid = os.getpid() - if threaded: - t = threading.current_thread() - # Thread objects in Python 2.4 and earlier do not have ident - # attrs. Worm around that. - ident = getattr(t, "ident", hash(t)) - self.tname = "-%x" % (ident & 0xffffffff) - else: - self.tname = "" - dirname = os.path.dirname(self.lock_file) - - # unique name is mostly about the current process, but must - # also contain the path -- otherwise, two adjacent locked - # files conflict (one file gets locked, creating lock-file and - # unique file, the other one gets locked, creating lock-file - # and overwriting the already existing lock-file, then one - # gets unlocked, deleting both lock-file and unique file, - # finally the last lock errors out upon releasing. - self.unique_name = os.path.join(dirname, - "%s%s.%s%s" % (self.hostname, - self.tname, - self.pid, - hash(self.path))) - self.timeout = timeout - - def is_locked(self): - """ - Tell whether or not the file is locked. - """ - raise NotImplemented("implement in subclass") - - def i_am_locking(self): - """ - Return True if this object is locking the file. - """ - raise NotImplemented("implement in subclass") - - def break_lock(self): - """ - Remove a lock. Useful if a locking thread failed to unlock. - """ - raise NotImplemented("implement in subclass") - - def __repr__(self): - return "<%s: %r -- %r>" % (self.__class__.__name__, self.unique_name, - self.path) - - -def _fl_helper(cls, mod, *args, **kwds): - warnings.warn("Import from %s module instead of lockfile package" % mod, - DeprecationWarning, stacklevel=2) - # This is a bit funky, but it's only for awhile. The way the unit tests - # are constructed this function winds up as an unbound method, so it - # actually takes three args, not two. We want to toss out self. - if not isinstance(args[0], str): - # We are testing, avoid the first arg - args = args[1:] - if len(args) == 1 and not kwds: - kwds["threaded"] = True - return cls(*args, **kwds) - - -def LinkFileLock(*args, **kwds): - """Factory function provided for backwards compatibility. - - Do not use in new code. Instead, import LinkLockFile from the - lockfile.linklockfile module. - """ - from . import linklockfile - return _fl_helper(linklockfile.LinkLockFile, "lockfile.linklockfile", - *args, **kwds) - - -def MkdirFileLock(*args, **kwds): - """Factory function provided for backwards compatibility. - - Do not use in new code. Instead, import MkdirLockFile from the - lockfile.mkdirlockfile module. - """ - from . import mkdirlockfile - return _fl_helper(mkdirlockfile.MkdirLockFile, "lockfile.mkdirlockfile", - *args, **kwds) - - -def SQLiteFileLock(*args, **kwds): - """Factory function provided for backwards compatibility. - - Do not use in new code. Instead, import SQLiteLockFile from the - lockfile.mkdirlockfile module. - """ - from . import sqlitelockfile - return _fl_helper(sqlitelockfile.SQLiteLockFile, "lockfile.sqlitelockfile", - *args, **kwds) - - -def locked(path, timeout=None): - """Decorator which enables locks for decorated function. - - Arguments: - - path: path for lockfile. - - timeout (optional): Timeout for acquiring lock. - - Usage: - @locked('/var/run/myname', timeout=0) - def myname(...): - ... - """ - def decor(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - lock = FileLock(path, timeout=timeout) - lock.acquire() - try: - return func(*args, **kwargs) - finally: - lock.release() - return wrapper - return decor - - -if hasattr(os, "link"): - from . import linklockfile as _llf - LockFile = _llf.LinkLockFile -else: - from . import mkdirlockfile as _mlf - LockFile = _mlf.MkdirLockFile - -FileLock = LockFile diff --git a/src/pip/_vendor/lockfile/linklockfile.py b/src/pip/_vendor/lockfile/linklockfile.py deleted file mode 100644 index 2ca9be04235..00000000000 --- a/src/pip/_vendor/lockfile/linklockfile.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import absolute_import - -import time -import os - -from . import (LockBase, LockFailed, NotLocked, NotMyLock, LockTimeout, - AlreadyLocked) - - -class LinkLockFile(LockBase): - """Lock access to a file using atomic property of link(2). - - >>> lock = LinkLockFile('somefile') - >>> lock = LinkLockFile('somefile', threaded=False) - """ - - def acquire(self, timeout=None): - try: - open(self.unique_name, "wb").close() - except IOError: - raise LockFailed("failed to create %s" % self.unique_name) - - timeout = timeout if timeout is not None else self.timeout - end_time = time.time() - if timeout is not None and timeout > 0: - end_time += timeout - - while True: - # Try and create a hard link to it. - try: - os.link(self.unique_name, self.lock_file) - except OSError: - # Link creation failed. Maybe we've double-locked? - nlinks = os.stat(self.unique_name).st_nlink - if nlinks == 2: - # The original link plus the one I created == 2. We're - # good to go. - return - else: - # Otherwise the lock creation failed. - if timeout is not None and time.time() > end_time: - os.unlink(self.unique_name) - if timeout > 0: - raise LockTimeout("Timeout waiting to acquire" - " lock for %s" % - self.path) - else: - raise AlreadyLocked("%s is already locked" % - self.path) - time.sleep(timeout is not None and timeout / 10 or 0.1) - else: - # Link creation succeeded. We're good to go. - return - - def release(self): - if not self.is_locked(): - raise NotLocked("%s is not locked" % self.path) - elif not os.path.exists(self.unique_name): - raise NotMyLock("%s is locked, but not by me" % self.path) - os.unlink(self.unique_name) - os.unlink(self.lock_file) - - def is_locked(self): - return os.path.exists(self.lock_file) - - def i_am_locking(self): - return (self.is_locked() and - os.path.exists(self.unique_name) and - os.stat(self.unique_name).st_nlink == 2) - - def break_lock(self): - if os.path.exists(self.lock_file): - os.unlink(self.lock_file) diff --git a/src/pip/_vendor/lockfile/mkdirlockfile.py b/src/pip/_vendor/lockfile/mkdirlockfile.py deleted file mode 100644 index 05a8c96ca51..00000000000 --- a/src/pip/_vendor/lockfile/mkdirlockfile.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import absolute_import, division - -import time -import os -import sys -import errno - -from . import (LockBase, LockFailed, NotLocked, NotMyLock, LockTimeout, - AlreadyLocked) - - -class MkdirLockFile(LockBase): - """Lock file by creating a directory.""" - def __init__(self, path, threaded=True, timeout=None): - """ - >>> lock = MkdirLockFile('somefile') - >>> lock = MkdirLockFile('somefile', threaded=False) - """ - LockBase.__init__(self, path, threaded, timeout) - # Lock file itself is a directory. Place the unique file name into - # it. - self.unique_name = os.path.join(self.lock_file, - "%s.%s%s" % (self.hostname, - self.tname, - self.pid)) - - def acquire(self, timeout=None): - timeout = timeout if timeout is not None else self.timeout - end_time = time.time() - if timeout is not None and timeout > 0: - end_time += timeout - - if timeout is None: - wait = 0.1 - else: - wait = max(0, timeout / 10) - - while True: - try: - os.mkdir(self.lock_file) - except OSError: - err = sys.exc_info()[1] - if err.errno == errno.EEXIST: - # Already locked. - if os.path.exists(self.unique_name): - # Already locked by me. - return - if timeout is not None and time.time() > end_time: - if timeout > 0: - raise LockTimeout("Timeout waiting to acquire" - " lock for %s" % - self.path) - else: - # Someone else has the lock. - raise AlreadyLocked("%s is already locked" % - self.path) - time.sleep(wait) - else: - # Couldn't create the lock for some other reason - raise LockFailed("failed to create %s" % self.lock_file) - else: - open(self.unique_name, "wb").close() - return - - def release(self): - if not self.is_locked(): - raise NotLocked("%s is not locked" % self.path) - elif not os.path.exists(self.unique_name): - raise NotMyLock("%s is locked, but not by me" % self.path) - os.unlink(self.unique_name) - os.rmdir(self.lock_file) - - def is_locked(self): - return os.path.exists(self.lock_file) - - def i_am_locking(self): - return (self.is_locked() and - os.path.exists(self.unique_name)) - - def break_lock(self): - if os.path.exists(self.lock_file): - for name in os.listdir(self.lock_file): - os.unlink(os.path.join(self.lock_file, name)) - os.rmdir(self.lock_file) diff --git a/src/pip/_vendor/lockfile/pidlockfile.py b/src/pip/_vendor/lockfile/pidlockfile.py deleted file mode 100644 index 069e85b15bd..00000000000 --- a/src/pip/_vendor/lockfile/pidlockfile.py +++ /dev/null @@ -1,190 +0,0 @@ -# -*- coding: utf-8 -*- - -# pidlockfile.py -# -# Copyright © 2008–2009 Ben Finney <ben+python@benfinney.id.au> -# -# This is free software: you may copy, modify, and/or distribute this work -# under the terms of the Python Software Foundation License, version 2 or -# later as published by the Python Software Foundation. -# No warranty expressed or implied. See the file LICENSE.PSF-2 for details. - -""" Lockfile behaviour implemented via Unix PID files. - """ - -from __future__ import absolute_import - -import errno -import os -import time - -from . import (LockBase, AlreadyLocked, LockFailed, NotLocked, NotMyLock, - LockTimeout) - - -class PIDLockFile(LockBase): - """ Lockfile implemented as a Unix PID file. - - The lock file is a normal file named by the attribute `path`. - A lock's PID file contains a single line of text, containing - the process ID (PID) of the process that acquired the lock. - - >>> lock = PIDLockFile('somefile') - >>> lock = PIDLockFile('somefile') - """ - - def __init__(self, path, threaded=False, timeout=None): - # pid lockfiles don't support threaded operation, so always force - # False as the threaded arg. - LockBase.__init__(self, path, False, timeout) - self.unique_name = self.path - - def read_pid(self): - """ Get the PID from the lock file. - """ - return read_pid_from_pidfile(self.path) - - def is_locked(self): - """ Test if the lock is currently held. - - The lock is held if the PID file for this lock exists. - - """ - return os.path.exists(self.path) - - def i_am_locking(self): - """ Test if the lock is held by the current process. - - Returns ``True`` if the current process ID matches the - number stored in the PID file. - """ - return self.is_locked() and os.getpid() == self.read_pid() - - def acquire(self, timeout=None): - """ Acquire the lock. - - Creates the PID file for this lock, or raises an error if - the lock could not be acquired. - """ - - timeout = timeout if timeout is not None else self.timeout - end_time = time.time() - if timeout is not None and timeout > 0: - end_time += timeout - - while True: - try: - write_pid_to_pidfile(self.path) - except OSError as exc: - if exc.errno == errno.EEXIST: - # The lock creation failed. Maybe sleep a bit. - if time.time() > end_time: - if timeout is not None and timeout > 0: - raise LockTimeout("Timeout waiting to acquire" - " lock for %s" % - self.path) - else: - raise AlreadyLocked("%s is already locked" % - self.path) - time.sleep(timeout is not None and timeout / 10 or 0.1) - else: - raise LockFailed("failed to create %s" % self.path) - else: - return - - def release(self): - """ Release the lock. - - Removes the PID file to release the lock, or raises an - error if the current process does not hold the lock. - - """ - if not self.is_locked(): - raise NotLocked("%s is not locked" % self.path) - if not self.i_am_locking(): - raise NotMyLock("%s is locked, but not by me" % self.path) - remove_existing_pidfile(self.path) - - def break_lock(self): - """ Break an existing lock. - - Removes the PID file if it already exists, otherwise does - nothing. - - """ - remove_existing_pidfile(self.path) - - -def read_pid_from_pidfile(pidfile_path): - """ Read the PID recorded in the named PID file. - - Read and return the numeric PID recorded as text in the named - PID file. If the PID file cannot be read, or if the content is - not a valid PID, return ``None``. - - """ - pid = None - try: - pidfile = open(pidfile_path, 'r') - except IOError: - pass - else: - # According to the FHS 2.3 section on PID files in /var/run: - # - # The file must consist of the process identifier in - # ASCII-encoded decimal, followed by a newline character. - # - # Programs that read PID files should be somewhat flexible - # in what they accept; i.e., they should ignore extra - # whitespace, leading zeroes, absence of the trailing - # newline, or additional lines in the PID file. - - line = pidfile.readline().strip() - try: - pid = int(line) - except ValueError: - pass - pidfile.close() - - return pid - - -def write_pid_to_pidfile(pidfile_path): - """ Write the PID in the named PID file. - - Get the numeric process ID (“PID”) of the current process - and write it to the named file as a line of text. - - """ - open_flags = (os.O_CREAT | os.O_EXCL | os.O_WRONLY) - open_mode = 0o644 - pidfile_fd = os.open(pidfile_path, open_flags, open_mode) - pidfile = os.fdopen(pidfile_fd, 'w') - - # According to the FHS 2.3 section on PID files in /var/run: - # - # The file must consist of the process identifier in - # ASCII-encoded decimal, followed by a newline character. For - # example, if crond was process number 25, /var/run/crond.pid - # would contain three characters: two, five, and newline. - - pid = os.getpid() - pidfile.write("%s\n" % pid) - pidfile.close() - - -def remove_existing_pidfile(pidfile_path): - """ Remove the named PID file if it exists. - - Removing a PID file that doesn't already exist puts us in the - desired state, so we ignore the condition if the file does not - exist. - - """ - try: - os.remove(pidfile_path) - except OSError as exc: - if exc.errno == errno.ENOENT: - pass - else: - raise diff --git a/src/pip/_vendor/lockfile/sqlitelockfile.py b/src/pip/_vendor/lockfile/sqlitelockfile.py deleted file mode 100644 index f997e2444e7..00000000000 --- a/src/pip/_vendor/lockfile/sqlitelockfile.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import absolute_import, division - -import time -import os - -try: - unicode -except NameError: - unicode = str - -from . import LockBase, NotLocked, NotMyLock, LockTimeout, AlreadyLocked - - -class SQLiteLockFile(LockBase): - "Demonstrate SQL-based locking." - - testdb = None - - def __init__(self, path, threaded=True, timeout=None): - """ - >>> lock = SQLiteLockFile('somefile') - >>> lock = SQLiteLockFile('somefile', threaded=False) - """ - LockBase.__init__(self, path, threaded, timeout) - self.lock_file = unicode(self.lock_file) - self.unique_name = unicode(self.unique_name) - - if SQLiteLockFile.testdb is None: - import tempfile - _fd, testdb = tempfile.mkstemp() - os.close(_fd) - os.unlink(testdb) - del _fd, tempfile - SQLiteLockFile.testdb = testdb - - import sqlite3 - self.connection = sqlite3.connect(SQLiteLockFile.testdb) - - c = self.connection.cursor() - try: - c.execute("create table locks" - "(" - " lock_file varchar(32)," - " unique_name varchar(32)" - ")") - except sqlite3.OperationalError: - pass - else: - self.connection.commit() - import atexit - atexit.register(os.unlink, SQLiteLockFile.testdb) - - def acquire(self, timeout=None): - timeout = timeout if timeout is not None else self.timeout - end_time = time.time() - if timeout is not None and timeout > 0: - end_time += timeout - - if timeout is None: - wait = 0.1 - elif timeout <= 0: - wait = 0 - else: - wait = timeout / 10 - - cursor = self.connection.cursor() - - while True: - if not self.is_locked(): - # Not locked. Try to lock it. - cursor.execute("insert into locks" - " (lock_file, unique_name)" - " values" - " (?, ?)", - (self.lock_file, self.unique_name)) - self.connection.commit() - - # Check to see if we are the only lock holder. - cursor.execute("select * from locks" - " where unique_name = ?", - (self.unique_name,)) - rows = cursor.fetchall() - if len(rows) > 1: - # Nope. Someone else got there. Remove our lock. - cursor.execute("delete from locks" - " where unique_name = ?", - (self.unique_name,)) - self.connection.commit() - else: - # Yup. We're done, so go home. - return - else: - # Check to see if we are the only lock holder. - cursor.execute("select * from locks" - " where unique_name = ?", - (self.unique_name,)) - rows = cursor.fetchall() - if len(rows) == 1: - # We're the locker, so go home. - return - - # Maybe we should wait a bit longer. - if timeout is not None and time.time() > end_time: - if timeout > 0: - # No more waiting. - raise LockTimeout("Timeout waiting to acquire" - " lock for %s" % - self.path) - else: - # Someone else has the lock and we are impatient.. - raise AlreadyLocked("%s is already locked" % self.path) - - # Well, okay. We'll give it a bit longer. - time.sleep(wait) - - def release(self): - if not self.is_locked(): - raise NotLocked("%s is not locked" % self.path) - if not self.i_am_locking(): - raise NotMyLock("%s is locked, but not by me (by %s)" % - (self.unique_name, self._who_is_locking())) - cursor = self.connection.cursor() - cursor.execute("delete from locks" - " where unique_name = ?", - (self.unique_name,)) - self.connection.commit() - - def _who_is_locking(self): - cursor = self.connection.cursor() - cursor.execute("select unique_name from locks" - " where lock_file = ?", - (self.lock_file,)) - return cursor.fetchone()[0] - - def is_locked(self): - cursor = self.connection.cursor() - cursor.execute("select * from locks" - " where lock_file = ?", - (self.lock_file,)) - rows = cursor.fetchall() - return not not rows - - def i_am_locking(self): - cursor = self.connection.cursor() - cursor.execute("select * from locks" - " where lock_file = ?" - " and unique_name = ?", - (self.lock_file, self.unique_name)) - return not not cursor.fetchall() - - def break_lock(self): - cursor = self.connection.cursor() - cursor.execute("delete from locks" - " where lock_file = ?", - (self.lock_file,)) - self.connection.commit() diff --git a/src/pip/_vendor/lockfile/symlinklockfile.py b/src/pip/_vendor/lockfile/symlinklockfile.py deleted file mode 100644 index 23b41f582b9..00000000000 --- a/src/pip/_vendor/lockfile/symlinklockfile.py +++ /dev/null @@ -1,70 +0,0 @@ -from __future__ import absolute_import - -import os -import time - -from . import (LockBase, NotLocked, NotMyLock, LockTimeout, - AlreadyLocked) - - -class SymlinkLockFile(LockBase): - """Lock access to a file using symlink(2).""" - - def __init__(self, path, threaded=True, timeout=None): - # super(SymlinkLockFile).__init(...) - LockBase.__init__(self, path, threaded, timeout) - # split it back! - self.unique_name = os.path.split(self.unique_name)[1] - - def acquire(self, timeout=None): - # Hopefully unnecessary for symlink. - # try: - # open(self.unique_name, "wb").close() - # except IOError: - # raise LockFailed("failed to create %s" % self.unique_name) - timeout = timeout if timeout is not None else self.timeout - end_time = time.time() - if timeout is not None and timeout > 0: - end_time += timeout - - while True: - # Try and create a symbolic link to it. - try: - os.symlink(self.unique_name, self.lock_file) - except OSError: - # Link creation failed. Maybe we've double-locked? - if self.i_am_locking(): - # Linked to out unique name. Proceed. - return - else: - # Otherwise the lock creation failed. - if timeout is not None and time.time() > end_time: - if timeout > 0: - raise LockTimeout("Timeout waiting to acquire" - " lock for %s" % - self.path) - else: - raise AlreadyLocked("%s is already locked" % - self.path) - time.sleep(timeout / 10 if timeout is not None else 0.1) - else: - # Link creation succeeded. We're good to go. - return - - def release(self): - if not self.is_locked(): - raise NotLocked("%s is not locked" % self.path) - elif not self.i_am_locking(): - raise NotMyLock("%s is locked, but not by me" % self.path) - os.unlink(self.lock_file) - - def is_locked(self): - return os.path.islink(self.lock_file) - - def i_am_locking(self): - return (os.path.islink(self.lock_file) - and os.readlink(self.lock_file) == self.unique_name) - - def break_lock(self): - if os.path.islink(self.lock_file): # exists && link - os.unlink(self.lock_file) diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 84ff34ea1f1..513f514aca4 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -6,7 +6,6 @@ distlib==0.2.9.post0 distro==1.4.0 html5lib==1.0.1 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -lockfile==0.12.2 msgpack==0.6.1 packaging==19.0 pep517==0.5.0 diff --git a/tests/conftest.py b/tests/conftest.py index ac1559788d1..5982ebf9014 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -74,7 +74,7 @@ def tmpdir_factory(request, tmpdir_factory): tmpdir_factory.getbasetemp().remove(ignore_errors=True) -@pytest.yield_fixture +@pytest.fixture def tmpdir(request, tmpdir): """ Return a temporary directory path object which is unique to each test @@ -227,7 +227,7 @@ def install_egg_link(venv, project_name, egg_info_dir): fp.write(str(egg_info_dir) + '\n.') -@pytest.yield_fixture(scope='session') +@pytest.fixture(scope='session') def virtualenv_template(request, tmpdir_factory, pip_src, setuptools_install, common_wheels): @@ -268,7 +268,7 @@ def virtualenv_template(request, tmpdir_factory, pip_src, yield venv -@pytest.yield_fixture +@pytest.fixture def virtualenv(virtualenv_template, tmpdir, isolate): """ Return a virtual environment which is unique to each test function diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 036a087aff7..7a0e4a9cbee 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -299,8 +299,7 @@ def test_install_editable_uninstalls_existing(data, script, tmpdir): 'install', '-e', '%s#egg=pip-test-package' % local_checkout( - 'git+https://github.com/pypa/pip-test-package.git', - tmpdir.joinpath("cache"), + 'git+https://github.com/pypa/pip-test-package.git', tmpdir, ), ) result.assert_installed('pip-test-package', with_files=['.git']) @@ -374,7 +373,7 @@ def test_vcs_url_urlquote_normalization(script, tmpdir): local_checkout( 'bzr+http://bazaar.launchpad.net/%7Edjango-wikiapp/django-wikiapp' '/release-0.1', - tmpdir.joinpath("cache"), + tmpdir, ), ) @@ -652,7 +651,7 @@ def test_install_using_install_option_and_editable(script, tmpdir): url = 'git+git://github.com/pypa/pip-test-package' result = script.pip( 'install', '-e', '%s#egg=pip-test-package' % - local_checkout(url, tmpdir.joinpath("cache")), + local_checkout(url, tmpdir), '--install-option=--script-dir=%s' % folder, expect_stderr=True) script_file = ( @@ -671,7 +670,7 @@ def test_install_global_option_using_editable(script, tmpdir): url = 'hg+http://bitbucket.org/runeh/anyjson' result = script.pip( 'install', '--global-option=--version', '-e', - '%s@0.2.5#egg=anyjson' % local_checkout(url, tmpdir.joinpath("cache")), + '%s@0.2.5#egg=anyjson' % local_checkout(url, tmpdir), expect_stderr=True) assert 'Successfully installed anyjson' in result.stdout diff --git a/tests/functional/test_install_cleanup.py b/tests/functional/test_install_cleanup.py index 8489b860b3a..dc87bc3f76a 100644 --- a/tests/functional/test_install_cleanup.py +++ b/tests/functional/test_install_cleanup.py @@ -47,10 +47,7 @@ def test_cleanup_after_install_editable_from_hg(script, tmpdir): 'install', '-e', '%s#egg=ScriptTest' % - local_checkout( - 'hg+https://bitbucket.org/ianb/scripttest', - tmpdir.joinpath("cache"), - ), + local_checkout('hg+https://bitbucket.org/ianb/scripttest', tmpdir), ) build = script.venv_path / 'build' src = script.venv_path / 'src' diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index 63f163ca97a..b906c37b642 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -111,8 +111,7 @@ def test_multiple_requirements_files(script, tmpdir): """) % ( local_checkout( - 'svn+http://svn.colorstudy.com/INITools/trunk', - tmpdir.joinpath("cache"), + 'svn+http://svn.colorstudy.com/INITools/trunk', tmpdir, ), other_lib_name ), diff --git a/tests/functional/test_install_upgrade.py b/tests/functional/test_install_upgrade.py index 91f81e168df..36b518b2546 100644 --- a/tests/functional/test_install_upgrade.py +++ b/tests/functional/test_install_upgrade.py @@ -349,8 +349,7 @@ def test_install_with_ignoreinstalled_requested(script): def test_upgrade_vcs_req_with_no_dists_found(script, tmpdir): """It can upgrade a VCS requirement that has no distributions otherwise.""" req = "%s#egg=pip-test-package" % local_checkout( - "git+https://github.com/pypa/pip-test-package.git", - tmpdir.joinpath("cache"), + "git+https://github.com/pypa/pip-test-package.git", tmpdir, ) script.pip("install", req) result = script.pip("install", "-U", req) diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index b07c1c9cbbb..d668ed12cb0 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -52,8 +52,7 @@ def test_install_subversion_usersite_editable_with_distribute( 'install', '--user', '-e', '%s#egg=initools' % local_checkout( - 'svn+http://svn.colorstudy.com/INITools/trunk', - tmpdir.joinpath("cache"), + 'svn+http://svn.colorstudy.com/INITools/trunk', tmpdir, ) ) result.assert_installed('INITools', use_user_site=True) diff --git a/tests/functional/test_install_vcs_git.py b/tests/functional/test_install_vcs_git.py index 11b63a84488..feb9ef0401e 100644 --- a/tests/functional/test_install_vcs_git.py +++ b/tests/functional/test_install_vcs_git.py @@ -58,7 +58,7 @@ def _github_checkout(url_path, temp_dir, rev=None, egg=None, scheme=None): if scheme is None: scheme = 'https' url = 'git+{}://github.com/{}'.format(scheme, url_path) - local_url = local_checkout(url, temp_dir.joinpath('cache')) + local_url = local_checkout(url, temp_dir) if rev is not None: local_url += '@{}'.format(rev) if egg is not None: diff --git a/tests/functional/test_uninstall.py b/tests/functional/test_uninstall.py index 128d66102c0..8ea54224677 100644 --- a/tests/functional/test_uninstall.py +++ b/tests/functional/test_uninstall.py @@ -296,8 +296,7 @@ def test_uninstall_editable_from_svn(script, tmpdir): result = script.pip( 'install', '-e', '%s#egg=initools' % local_checkout( - 'svn+http://svn.colorstudy.com/INITools/trunk', - tmpdir.joinpath("cache"), + 'svn+http://svn.colorstudy.com/INITools/trunk', tmpdir, ), ) result.assert_installed('INITools') @@ -318,34 +317,29 @@ def test_uninstall_editable_from_svn(script, tmpdir): def test_uninstall_editable_with_source_outside_venv(script, tmpdir): """ Test uninstalling editable install from existing source outside the venv. - """ - cache_dir = tmpdir.joinpath("cache") - try: temp = mkdtemp() - tmpdir = join(temp, 'pip-test-package') + temp_pkg_dir = join(temp, 'pip-test-package') _test_uninstall_editable_with_source_outside_venv( script, tmpdir, - cache_dir, + temp_pkg_dir, ) finally: rmtree(temp) def _test_uninstall_editable_with_source_outside_venv( - script, tmpdir, cache_dir): + script, tmpdir, temp_pkg_dir, +): result = script.run( 'git', 'clone', - local_repo( - 'git+git://github.com/pypa/pip-test-package', - cache_dir, - ), - tmpdir, + local_repo('git+git://github.com/pypa/pip-test-package', tmpdir), + temp_pkg_dir, expect_stderr=True, ) - result2 = script.pip('install', '-e', tmpdir) + result2 = script.pip('install', '-e', temp_pkg_dir) assert join( script.site_packages, 'pip-test-package.egg-link' ) in result2.files_created, list(result2.files_created.keys()) @@ -370,10 +364,7 @@ def test_uninstall_from_reqs_file(script, tmpdir): # and something else to test out: PyLogo<0.4 """) % - local_checkout( - 'svn+http://svn.colorstudy.com/INITools/trunk', - tmpdir.joinpath("cache") - ) + local_checkout('svn+http://svn.colorstudy.com/INITools/trunk', tmpdir) ) result = script.pip('install', '-r', 'test-req.txt') script.scratch_path.joinpath("test-req.txt").write_text( @@ -387,10 +378,7 @@ def test_uninstall_from_reqs_file(script, tmpdir): # and something else to test out: PyLogo<0.4 """) % - local_checkout( - 'svn+http://svn.colorstudy.com/INITools/trunk', - tmpdir.joinpath("cache") - ) + local_checkout('svn+http://svn.colorstudy.com/INITools/trunk', tmpdir) ) result2 = script.pip('uninstall', '-r', 'test-req.txt', '-y') assert_all_changes( @@ -458,6 +446,28 @@ def test_uninstall_wheel(script, data): assert_all_changes(result, result2, []) +@pytest.mark.skipif("sys.platform == 'win32'") +def test_uninstall_with_symlink(script, data, tmpdir): + """ + Test uninstalling a wheel, with an additional symlink + https://github.com/pypa/pip/issues/6892 + """ + package = data.packages.joinpath("simple.dist-0.1-py2.py3-none-any.whl") + script.pip('install', package, '--no-index') + symlink_target = tmpdir / "target" + symlink_target.mkdir() + symlink_source = script.site_packages / "symlink" + (script.base_path / symlink_source).symlink_to(symlink_target) + st_mode = symlink_target.stat().st_mode + distinfo_path = script.site_packages_path / 'simple.dist-0.1.dist-info' + record_path = distinfo_path / 'RECORD' + with open(record_path, "a") as f: + f.write("symlink,,\n") + uninstall_result = script.pip('uninstall', 'simple.dist', '-y') + assert symlink_source in uninstall_result.files_deleted + assert symlink_target.stat().st_mode == st_mode + + def test_uninstall_setuptools_develop_install(script, data): """Try uninstall after setup.py develop followed of setup.py install""" pkg_path = data.packages.joinpath("FSPkg") diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index b7046b34be6..a1ddd23f1d0 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -13,6 +13,7 @@ import pytest from scripttest import FoundDir, TestFileEnvironment +from pip._internal.collector import LinkCollector from pip._internal.download import PipSession from pip._internal.index import PackageFinder from pip._internal.locations import get_major_minor_version @@ -89,11 +90,28 @@ def make_test_search_scope( if index_urls is None: index_urls = [] - return SearchScope.create( + return SearchScope.create(find_links=find_links, index_urls=index_urls) + + +def make_test_link_collector( + find_links=None, # type: Optional[List[str]] + index_urls=None, # type: Optional[List[str]] + session=None, # type: Optional[PipSession] +): + # type: (...) -> LinkCollector + """ + Create a LinkCollector object for testing purposes. + """ + if session is None: + session = PipSession() + + search_scope = make_test_search_scope( find_links=find_links, index_urls=index_urls, ) + return LinkCollector(session=session, search_scope=search_scope) + def make_test_finder( find_links=None, # type: Optional[List[str]] @@ -106,12 +124,10 @@ def make_test_finder( """ Create a PackageFinder for testing purposes. """ - if session is None: - session = PipSession() - - search_scope = make_test_search_scope( + link_collector = make_test_link_collector( find_links=find_links, index_urls=index_urls, + session=session, ) selection_prefs = SelectionPreferences( allow_yanked=True, @@ -119,9 +135,8 @@ def make_test_finder( ) return PackageFinder.create( - search_scope=search_scope, + link_collector=link_collector, selection_prefs=selection_prefs, - session=session, target_python=target_python, ) diff --git a/tests/lib/local_repos.py b/tests/lib/local_repos.py index 69c60adb3fb..40cb19b678a 100644 --- a/tests/lib/local_repos.py +++ b/tests/lib/local_repos.py @@ -6,9 +6,13 @@ from pip._vendor.six.moves.urllib import request as urllib_request from pip._internal.utils.misc import hide_url +from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.vcs import bazaar, git, mercurial, subversion from tests.lib import path_to_url +if MYPY_CHECK_RUNNING: + from tests.lib.path import Path + def _create_initools_repository(directory): subprocess.check_call('svnadmin create INITools'.split(), cwd=directory) @@ -68,7 +72,17 @@ def _get_vcs_and_checkout_url(remote_repository, directory): ) -def local_checkout(remote_repo, directory): +def local_checkout( + remote_repo, # type: str + temp_path, # type: Path +): + # type: (...) -> str + """ + :param temp_path: the return value of the tmpdir fixture, which is a + temp directory Path object unique to each test function invocation, + created as a sub directory of the base temp directory. + """ + directory = temp_path.joinpath('cache') if not os.path.exists(directory): os.mkdir(directory) # os.makedirs(directory) @@ -78,5 +92,5 @@ def local_checkout(remote_repo, directory): return _get_vcs_and_checkout_url(remote_repo, directory) -def local_repo(remote_repo, directory): - return local_checkout(remote_repo, directory).split('+', 1)[1] +def local_repo(remote_repo, temp_path): + return local_checkout(remote_repo, temp_path).split('+', 1)[1] diff --git a/tests/lib/path.py b/tests/lib/path.py index cb2e6bda7ed..b2676a2e1e0 100644 --- a/tests/lib/path.py +++ b/tests/lib/path.py @@ -208,4 +208,10 @@ def touch(self): path = fp.fileno() if os.utime in supports_fd else self os.utime(path, None) # times is not optional on Python 2.7 + def symlink_to(self, target): + os.symlink(target, self) + + def stat(self): + return os.stat(self) + curdir = Path(os.path.curdir) diff --git a/tests/unit/test_build_env.py b/tests/unit/test_build_env.py index 9bfd19aa281..3e3c7ce9fcb 100644 --- a/tests/unit/test_build_env.py +++ b/tests/unit/test_build_env.py @@ -22,6 +22,7 @@ def run_with_build_env(script, setup_script_contents, import sys from pip._internal.build_env import BuildEnvironment + from pip._internal.collector import LinkCollector from pip._internal.download import PipSession from pip._internal.index import PackageFinder from pip._internal.models.search_scope import SearchScope @@ -29,14 +30,16 @@ def run_with_build_env(script, setup_script_contents, SelectionPreferences ) - search_scope = SearchScope.create([%r], []) + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope.create([%r], []), + ) selection_prefs = SelectionPreferences( allow_yanked=True, ) finder = PackageFinder.create( - search_scope, + link_collector=link_collector, selection_prefs=selection_prefs, - session=PipSession(), ) build_env = BuildEnvironment() diff --git a/tests/unit/test_cmdoptions.py b/tests/unit/test_cmdoptions.py index 3cf2ba8b9ab..150570e716e 100644 --- a/tests/unit/test_cmdoptions.py +++ b/tests/unit/test_cmdoptions.py @@ -1,75 +1,6 @@ -import os - -import pretend import pytest -from mock import patch - -from pip._internal.cli.cmdoptions import ( - _convert_python_version, - make_search_scope, -) - - -@pytest.mark.parametrize( - 'find_links, no_index, suppress_no_index, expected', [ - (['link1'], False, False, - (['link1'], ['default_url', 'url1', 'url2'])), - (['link1'], False, True, (['link1'], ['default_url', 'url1', 'url2'])), - (['link1'], True, False, (['link1'], [])), - # Passing suppress_no_index=True suppresses no_index=True. - (['link1'], True, True, (['link1'], ['default_url', 'url1', 'url2'])), - # Test options.find_links=False. - (False, False, False, ([], ['default_url', 'url1', 'url2'])), - ], -) -def test_make_search_scope(find_links, no_index, suppress_no_index, expected): - """ - :param expected: the expected (find_links, index_urls) values. - """ - expected_find_links, expected_index_urls = expected - options = pretend.stub( - find_links=find_links, - index_url='default_url', - extra_index_urls=['url1', 'url2'], - no_index=no_index, - ) - search_scope = make_search_scope( - options, suppress_no_index=suppress_no_index, - ) - assert search_scope.find_links == expected_find_links - assert search_scope.index_urls == expected_index_urls - - -@patch('pip._internal.utils.misc.expanduser') -def test_make_search_scope__find_links_expansion(mock_expanduser, tmpdir): - """ - Test "~" expansion in --find-links paths. - """ - # This is a mock version of expanduser() that expands "~" to the tmpdir. - def expand_path(path): - if path.startswith('~/'): - path = os.path.join(tmpdir, path[2:]) - return path - - mock_expanduser.side_effect = expand_path - - options = pretend.stub( - find_links=['~/temp1', '~/temp2'], - index_url='default_url', - extra_index_urls=[], - no_index=False, - ) - # Only create temp2 and not temp1 to test that "~" expansion only occurs - # when the directory exists. - temp2_dir = os.path.join(tmpdir, 'temp2') - os.mkdir(temp2_dir) - - search_scope = make_search_scope(options) - # Only ~/temp2 gets expanded. Also, the path is normalized when expanded. - expected_temp2_dir = os.path.normcase(temp2_dir) - assert search_scope.find_links == ['~/temp1', expected_temp2_dir] - assert search_scope.index_urls == ['default_url'] +from pip._internal.cli.cmdoptions import _convert_python_version @pytest.mark.parametrize('value, expected', [ diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py new file mode 100644 index 00000000000..1e90948ebce --- /dev/null +++ b/tests/unit/test_collector.py @@ -0,0 +1,429 @@ +import logging +import os.path +from textwrap import dedent + +import mock +import pytest +from mock import Mock, patch +from pip._vendor import html5lib, requests +from pip._vendor.six.moves.urllib import request as urllib_request + +from pip._internal.collector import ( + HTMLPage, + _clean_link, + _determine_base_url, + _get_html_page, + _get_html_response, + _NotHTML, + _NotHTTP, + group_locations, +) +from pip._internal.download import PipSession +from pip._internal.models.index import PyPI +from pip._internal.models.link import Link +from tests.lib import make_test_link_collector + + +@pytest.mark.parametrize( + "url", + [ + "ftp://python.org/python-3.7.1.zip", + "file:///opt/data/pip-18.0.tar.gz", + ], +) +def test_get_html_response_archive_to_naive_scheme(url): + """ + `_get_html_response()` should error on an archive-like URL if the scheme + does not allow "poking" without getting data. + """ + with pytest.raises(_NotHTTP): + _get_html_response(url, session=mock.Mock(PipSession)) + + +@pytest.mark.parametrize( + "url, content_type", + [ + ("http://python.org/python-3.7.1.zip", "application/zip"), + ("https://pypi.org/pip-18.0.tar.gz", "application/gzip"), + ], +) +def test_get_html_response_archive_to_http_scheme(url, content_type): + """ + `_get_html_response()` should send a HEAD request on an archive-like URL + if the scheme supports it, and raise `_NotHTML` if the response isn't HTML. + """ + session = mock.Mock(PipSession) + session.head.return_value = mock.Mock(**{ + "request.method": "HEAD", + "headers": {"Content-Type": content_type}, + }) + + with pytest.raises(_NotHTML) as ctx: + _get_html_response(url, session=session) + + session.assert_has_calls([ + mock.call.head(url, allow_redirects=True), + ]) + assert ctx.value.args == (content_type, "HEAD") + + +@pytest.mark.parametrize( + "url", + [ + "http://python.org/python-3.7.1.zip", + "https://pypi.org/pip-18.0.tar.gz", + ], +) +def test_get_html_response_archive_to_http_scheme_is_html(url): + """ + `_get_html_response()` should work with archive-like URLs if the HEAD + request is responded with text/html. + """ + session = mock.Mock(PipSession) + session.head.return_value = mock.Mock(**{ + "request.method": "HEAD", + "headers": {"Content-Type": "text/html"}, + }) + session.get.return_value = mock.Mock(headers={"Content-Type": "text/html"}) + + resp = _get_html_response(url, session=session) + + assert resp is not None + assert session.mock_calls == [ + mock.call.head(url, allow_redirects=True), + mock.call.head().raise_for_status(), + mock.call.get(url, headers={ + "Accept": "text/html", "Cache-Control": "max-age=0", + }), + mock.call.get().raise_for_status(), + ] + + +@pytest.mark.parametrize( + "url", + [ + "https://pypi.org/simple/pip", + "https://pypi.org/simple/pip/", + "https://python.org/sitemap.xml", + ], +) +def test_get_html_response_no_head(url): + """ + `_get_html_response()` shouldn't send a HEAD request if the URL does not + look like an archive, only the GET request that retrieves data. + """ + session = mock.Mock(PipSession) + + # Mock the headers dict to ensure it is accessed. + session.get.return_value = mock.Mock(headers=mock.Mock(**{ + "get.return_value": "text/html", + })) + + resp = _get_html_response(url, session=session) + + assert resp is not None + assert session.head.call_count == 0 + assert session.get.mock_calls == [ + mock.call(url, headers={ + "Accept": "text/html", "Cache-Control": "max-age=0", + }), + mock.call().raise_for_status(), + mock.call().headers.get("Content-Type", ""), + ] + + +def test_get_html_response_dont_log_clear_text_password(caplog): + """ + `_get_html_response()` should redact the password from the index URL + in its DEBUG log message. + """ + session = mock.Mock(PipSession) + + # Mock the headers dict to ensure it is accessed. + session.get.return_value = mock.Mock(headers=mock.Mock(**{ + "get.return_value": "text/html", + })) + + caplog.set_level(logging.DEBUG) + + resp = _get_html_response( + "https://user:my_password@example.com/simple/", session=session + ) + + assert resp is not None + + assert len(caplog.records) == 1 + record = caplog.records[0] + assert record.levelname == 'DEBUG' + assert record.message.splitlines() == [ + "Getting page https://user:****@example.com/simple/", + ] + + +@pytest.mark.parametrize( + ("html", "url", "expected"), + [ + (b"<html></html>", "https://example.com/", "https://example.com/"), + ( + b"<html><head>" + b"<base href=\"https://foo.example.com/\">" + b"</head></html>", + "https://example.com/", + "https://foo.example.com/", + ), + ( + b"<html><head>" + b"<base><base href=\"https://foo.example.com/\">" + b"</head></html>", + "https://example.com/", + "https://foo.example.com/", + ), + ], +) +def test_determine_base_url(html, url, expected): + document = html5lib.parse( + html, transport_encoding=None, namespaceHTMLElements=False, + ) + assert _determine_base_url(document, url) == expected + + +@pytest.mark.parametrize( + ("url", "clean_url"), + [ + # URL with hostname and port. Port separator should not be quoted. + ("https://localhost.localdomain:8181/path/with space/", + "https://localhost.localdomain:8181/path/with%20space/"), + # URL that is already properly quoted. The quoting `%` + # characters should not be quoted again. + ("https://localhost.localdomain:8181/path/with%20quoted%20space/", + "https://localhost.localdomain:8181/path/with%20quoted%20space/"), + # URL with IPv4 address and port. + ("https://127.0.0.1:8181/path/with space/", + "https://127.0.0.1:8181/path/with%20space/"), + # URL with IPv6 address and port. The `[]` brackets around the + # IPv6 address should not be quoted. + ("https://[fd00:0:0:236::100]:8181/path/with space/", + "https://[fd00:0:0:236::100]:8181/path/with%20space/"), + # URL with query. The leading `?` should not be quoted. + ("https://localhost.localdomain:8181/path/with/query?request=test", + "https://localhost.localdomain:8181/path/with/query?request=test"), + # URL with colon in the path portion. + ("https://localhost.localdomain:8181/path:/with:/colon", + "https://localhost.localdomain:8181/path%3A/with%3A/colon"), + # URL with something that looks like a drive letter, but is + # not. The `:` should be quoted. + ("https://localhost.localdomain/T:/path/", + "https://localhost.localdomain/T%3A/path/"), + # VCS URL containing revision string. + ("git+ssh://example.com/path to/repo.git@1.0#egg=my-package-1.0", + "git+ssh://example.com/path%20to/repo.git@1.0#egg=my-package-1.0"), + # URL with Windows drive letter. The `:` after the drive + # letter should not be quoted. The trailing `/` should be + # removed. + pytest.param( + "file:///T:/path/with spaces/", + "file:///T:/path/with%20spaces", + marks=pytest.mark.skipif("sys.platform != 'win32'"), + ), + # URL with Windows drive letter, running on non-windows + # platform. The `:` after the drive should be quoted. + pytest.param( + "file:///T:/path/with spaces/", + "file:///T%3A/path/with%20spaces/", + marks=pytest.mark.skipif("sys.platform == 'win32'"), + ), + ] +) +def test_clean_link(url, clean_url): + assert(_clean_link(url) == clean_url) + + +class TestHTMLPage: + + @pytest.mark.parametrize( + ('anchor_html, expected'), + [ + # Test not present. + ('<a href="/pkg1-1.0.tar.gz"></a>', None), + # Test present with no value. + ('<a href="/pkg2-1.0.tar.gz" data-yanked></a>', ''), + # Test the empty string. + ('<a href="/pkg3-1.0.tar.gz" data-yanked=""></a>', ''), + # Test a non-empty string. + ('<a href="/pkg4-1.0.tar.gz" data-yanked="error"></a>', 'error'), + # Test a value with an escaped character. + ('<a href="/pkg4-1.0.tar.gz" data-yanked="version < 1"></a>', + 'version < 1'), + # Test a yanked reason with a non-ascii character. + (u'<a href="/pkg-1.0.tar.gz" data-yanked="curlyquote \u2018"></a>', + u'curlyquote \u2018'), + ] + ) + def test_iter_links__yanked_reason(self, anchor_html, expected): + html = ( + # Mark this as a unicode string for Python 2 since anchor_html + # can contain non-ascii. + u'<html><head><meta charset="utf-8"><head>' + '<body>{}</body></html>' + ).format(anchor_html) + html_bytes = html.encode('utf-8') + page = HTMLPage(html_bytes, url='https://example.com/simple/') + links = list(page.iter_links()) + link, = links + actual = link.yanked_reason + assert actual == expected + + +def test_request_http_error(caplog): + caplog.set_level(logging.DEBUG) + link = Link('http://localhost') + session = Mock(PipSession) + session.get.return_value = resp = Mock() + resp.raise_for_status.side_effect = requests.HTTPError('Http error') + assert _get_html_page(link, session=session) is None + assert ( + 'Could not fetch URL http://localhost: Http error - skipping' + in caplog.text + ) + + +def test_request_retries(caplog): + caplog.set_level(logging.DEBUG) + link = Link('http://localhost') + session = Mock(PipSession) + session.get.side_effect = requests.exceptions.RetryError('Retry error') + assert _get_html_page(link, session=session) is None + assert ( + 'Could not fetch URL http://localhost: Retry error - skipping' + in caplog.text + ) + + +@pytest.mark.parametrize( + "url, vcs_scheme", + [ + ("svn+http://pypi.org/something", "svn"), + ("git+https://github.com/pypa/pip.git", "git"), + ], +) +def test_get_html_page_invalid_scheme(caplog, url, vcs_scheme): + """`_get_html_page()` should error if an invalid scheme is given. + + Only file:, http:, https:, and ftp: are allowed. + """ + with caplog.at_level(logging.DEBUG): + page = _get_html_page(Link(url), session=mock.Mock(PipSession)) + + assert page is None + assert caplog.record_tuples == [ + ( + "pip._internal.collector", + logging.DEBUG, + "Cannot look at {} URL {}".format(vcs_scheme, url), + ), + ] + + +def test_get_html_page_directory_append_index(tmpdir): + """`_get_html_page()` should append "index.html" to a directory URL. + """ + dirpath = tmpdir.mkdir("something") + dir_url = "file:///{}".format( + urllib_request.pathname2url(dirpath).lstrip("/"), + ) + + session = mock.Mock(PipSession) + with mock.patch("pip._internal.collector._get_html_response") as mock_func: + _get_html_page(Link(dir_url), session=session) + assert mock_func.mock_calls == [ + mock.call( + "{}/index.html".format(dir_url.rstrip("/")), + session=session, + ), + ] + + +def test_group_locations__file_expand_dir(data): + """ + Test that a file:// dir gets listdir run with expand_dir + """ + files, urls = group_locations([data.find_links], expand_dir=True) + assert files and not urls, ( + "files and not urls should have been found at find-links url: %s" % + data.find_links + ) + + +def test_group_locations__file_not_find_link(data): + """ + Test that a file:// url dir that's not a find-link, doesn't get a listdir + run + """ + files, urls = group_locations([data.index_url("empty_with_pkg")]) + assert urls and not files, "urls, but not files should have been found" + + +def test_group_locations__non_existing_path(): + """ + Test that a non-existing path is ignored. + """ + files, urls = group_locations([os.path.join('this', 'doesnt', 'exist')]) + assert not urls and not files, "nothing should have been found" + + +def make_fake_html_page(url): + html = dedent(u"""\ + <html><head><meta name="api-version" value="2" /></head> + <body> + <a href="/abc-1.0.tar.gz#md5=000000000">abc-1.0.tar.gz</a> + </body></html> + """) + content = html.encode('utf-8') + headers = {} + return HTMLPage(content, url=url, headers=headers) + + +def check_links_include(links, names): + """ + Assert that the given list of Link objects includes, for each of the + given names, a link whose URL has a base name matching that name. + """ + for name in names: + assert any(link.url.endswith(name) for link in links), ( + 'name {!r} not among links: {}'.format(name, links) + ) + + +class TestLinkCollector(object): + + @patch('pip._internal.collector._get_html_response') + def test_collect_links(self, mock_get_html_response, data): + expected_url = 'https://pypi.org/simple/twine/' + + fake_page = make_fake_html_page(expected_url) + mock_get_html_response.return_value = fake_page + + link_collector = make_test_link_collector( + find_links=[data.find_links], + index_urls=[PyPI.simple_url], + ) + actual = link_collector.collect_links('twine') + + mock_get_html_response.assert_called_once_with( + expected_url, session=link_collector.session, + ) + + # Spot-check the CollectedLinks return value. + assert len(actual.files) > 20 + check_links_include(actual.files, names=['simple-1.0.tar.gz']) + + assert len(actual.find_links) == 1 + check_links_include(actual.find_links, names=['packages']) + + actual_pages = actual.pages + assert list(actual_pages) == [expected_url] + actual_page_links = actual_pages[expected_url] + assert len(actual_page_links) == 1 + assert actual_page_links[0].url == ( + 'https://pypi.org/abc-1.0.tar.gz#md5=000000000' + ) diff --git a/tests/unit/test_download.py b/tests/unit/test_download.py index 9c0ccc2cf30..87bc6a4ad27 100644 --- a/tests/unit/test_download.py +++ b/tests/unit/test_download.py @@ -10,6 +10,7 @@ import pytest from mock import Mock, patch +from pip._vendor.cachecontrol.caches import FileCache import pip from pip._internal.download import ( @@ -19,12 +20,10 @@ SafeFileCache, _copy_source_tree, _download_http_url, - _get_url_scheme, parse_content_disposition, sanitize_content_filename, unpack_file_url, unpack_http_url, - url_to_path, ) from pip._internal.exceptions import HashMismatch from pip._internal.models.link import Link @@ -301,48 +300,6 @@ def test_download_http_url__no_directory_traversal(tmpdir): assert actual == ['out_dir_file'] -@pytest.mark.parametrize("url,expected", [ - ('http://localhost:8080/', 'http'), - ('file:c:/path/to/file', 'file'), - ('file:/dev/null', 'file'), - ('', None), -]) -def test__get_url_scheme(url, expected): - assert _get_url_scheme(url) == expected - - -@pytest.mark.parametrize("url,win_expected,non_win_expected", [ - ('file:tmp', 'tmp', 'tmp'), - ('file:c:/path/to/file', r'C:\path\to\file', 'c:/path/to/file'), - ('file:/path/to/file', r'\path\to\file', '/path/to/file'), - ('file://localhost/tmp/file', r'\tmp\file', '/tmp/file'), - ('file://localhost/c:/tmp/file', r'C:\tmp\file', '/c:/tmp/file'), - ('file://somehost/tmp/file', r'\\somehost\tmp\file', None), - ('file:///tmp/file', r'\tmp\file', '/tmp/file'), - ('file:///c:/tmp/file', r'C:\tmp\file', '/c:/tmp/file'), -]) -def test_url_to_path(url, win_expected, non_win_expected): - if sys.platform == 'win32': - expected_path = win_expected - else: - expected_path = non_win_expected - - if expected_path is None: - with pytest.raises(ValueError): - url_to_path(url) - else: - assert url_to_path(url) == expected_path - - -@pytest.mark.skipif("sys.platform != 'win32'") -def test_url_to_path_path_to_url_symmetry_win(): - path = r'C:\tmp\file' - assert url_to_path(path_to_url(path)) == path - - unc_path = r'\\unc\share\path' - assert url_to_path(path_to_url(unc_path)) == unc_path - - @pytest.fixture def clean_project(tmpdir_factory, data): tmpdir = Path(str(tmpdir_factory.mktemp("clean_project"))) @@ -593,6 +550,14 @@ def test_safe_delete_no_perms(self, cache_tmpdir): cache = SafeFileCache(cache_tmpdir) cache.delete("foo") + def test_cache_hashes_are_same(self, cache_tmpdir): + cache = SafeFileCache(cache_tmpdir) + key = "test key" + fake_cache = Mock( + FileCache, directory=cache.directory, encode=FileCache.encode + ) + assert cache._get_cache_path(key) == FileCache._fn(fake_cache, key) + class TestPipSession: diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index fd68fcf8e27..1295ff0b059 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -37,7 +37,7 @@ def make_no_network_finder( ) # Replace the PackageFinder._link_collector's _get_pages() with a no-op. link_collector = finder._link_collector - link_collector._get_pages = lambda locations, project_name: [] + link_collector._get_pages = lambda locations: [] return finder @@ -62,7 +62,7 @@ def test_no_partial_name_match(data): def test_tilde(): """Finder can accept a path with ~ in it and will normalize it.""" - with patch('pip._internal.index.os.path.exists', return_value=True): + with patch('pip._internal.collector.os.path.exists', return_value=True): finder = make_test_finder(find_links=['~/python-pkgs']) req = install_req_from_line("gmpy") with pytest.raises(DistributionNotFound): diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py index d884fe52c08..47f3c67fcc8 100644 --- a/tests/unit/test_index.py +++ b/tests/unit/test_index.py @@ -1,119 +1,29 @@ import logging -import os.path -from textwrap import dedent import pytest -from mock import Mock, patch -from pip._vendor import html5lib, requests from pip._vendor.packaging.specifiers import SpecifierSet +from pip._internal.collector import LinkCollector from pip._internal.download import PipSession from pip._internal.index import ( CandidateEvaluator, CandidatePreferences, FormatControl, - HTMLPage, - LinkCollector, LinkEvaluator, PackageFinder, _check_link_requires_python, - _clean_link, - _determine_base_url, _extract_version_from_fragment, _find_name_version_sep, - _get_html_page, filter_unallowed_hashes, - group_locations, ) from pip._internal.models.candidate import InstallationCandidate -from pip._internal.models.index import PyPI from pip._internal.models.link import Link from pip._internal.models.search_scope import SearchScope from pip._internal.models.selection_prefs import SelectionPreferences from pip._internal.models.target_python import TargetPython from pip._internal.pep425tags import get_supported from pip._internal.utils.hashes import Hashes -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from tests.lib import CURRENT_PY_VERSION_INFO, make_test_search_scope - -if MYPY_CHECK_RUNNING: - from typing import List, Optional - - -def make_fake_html_page(url): - html = dedent(u"""\ - <html><head><meta name="api-version" value="2" /></head> - <body> - <a href="/abc-1.0.tar.gz#md5=000000000">abc-1.0.tar.gz</a> - </body></html> - """) - content = html.encode('utf-8') - headers = {} - return HTMLPage(content, url=url, headers=headers) - - -def make_test_link_collector( - find_links=None, # type: Optional[List[str]] -): - # type: (...) -> LinkCollector - """ - Create a LinkCollector object for testing purposes. - """ - session = PipSession() - search_scope = make_test_search_scope( - find_links=find_links, - index_urls=[PyPI.simple_url], - ) - - return LinkCollector( - session=session, - search_scope=search_scope, - ) - - -def check_links_include(links, names): - """ - Assert that the given list of Link objects includes, for each of the - given names, a link whose URL has a base name matching that name. - """ - for name in names: - assert any(link.url.endswith(name) for link in links), ( - 'name {!r} not among links: {}'.format(name, links) - ) - - -class TestLinkCollector(object): - - @patch('pip._internal.index._get_html_response') - def test_collect_links(self, mock_get_html_response, data): - expected_url = 'https://pypi.org/simple/twine/' - - fake_page = make_fake_html_page(expected_url) - mock_get_html_response.return_value = fake_page - - link_collector = make_test_link_collector( - find_links=[data.find_links] - ) - actual = link_collector.collect_links('twine') - - mock_get_html_response.assert_called_once_with( - expected_url, session=link_collector.session, - ) - - # Spot-check the CollectedLinks return value. - assert len(actual.files) > 20 - check_links_include(actual.files, names=['simple-1.0.tar.gz']) - - assert len(actual.find_links) == 1 - check_links_include(actual.find_links, names=['packages']) - - actual_pages = actual.pages - assert list(actual_pages) == [expected_url] - actual_page_links = actual_pages[expected_url] - assert len(actual_page_links) == 1 - assert actual_page_links[0].url == ( - 'https://pypi.org/abc-1.0.tar.gz#md5=000000000' - ) +from tests.lib import CURRENT_PY_VERSION_INFO def make_mock_candidate(version, yanked_reason=None, hex_digest=None): @@ -623,6 +533,7 @@ def test_sort_best_candidate__best_yanked_but_not_all( """ Test the best candidates being yanked, but not all. """ + caplog.set_level(logging.INFO) candidates = [ make_mock_candidate('4.0', yanked_reason='bad metadata #4'), # Put the best candidate in the middle, to test sorting. @@ -654,15 +565,18 @@ def test_create__candidate_prefs( """ Test that the _candidate_prefs attribute is set correctly. """ + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) selection_prefs = SelectionPreferences( allow_yanked=True, allow_all_prereleases=allow_all_prereleases, prefer_binary=prefer_binary, ) finder = PackageFinder.create( - search_scope=SearchScope([], []), + link_collector=link_collector, selection_prefs=selection_prefs, - session=PipSession(), ) candidate_prefs = finder._candidate_prefs assert candidate_prefs.allow_all_prereleases == allow_all_prereleases @@ -672,27 +586,29 @@ def test_create__link_collector(self): """ Test that the _link_collector attribute is set correctly. """ - search_scope = SearchScope([], []) - session = PipSession() + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) finder = PackageFinder.create( - search_scope=search_scope, + link_collector=link_collector, selection_prefs=SelectionPreferences(allow_yanked=True), - session=session, ) - actual_link_collector = finder._link_collector - assert actual_link_collector.search_scope is search_scope - assert actual_link_collector.session is session + assert finder._link_collector is link_collector def test_create__target_python(self): """ Test that the _target_python attribute is set correctly. """ + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) target_python = TargetPython(py_version_info=(3, 7, 3)) finder = PackageFinder.create( - search_scope=SearchScope([], []), + link_collector=link_collector, selection_prefs=SelectionPreferences(allow_yanked=True), - session=PipSession(), target_python=target_python, ) actual_target_python = finder._target_python @@ -705,10 +621,13 @@ def test_create__target_python_none(self): """ Test passing target_python=None. """ - finder = PackageFinder.create( + link_collector = LinkCollector( + session=PipSession(), search_scope=SearchScope([], []), + ) + finder = PackageFinder.create( + link_collector=link_collector, selection_prefs=SelectionPreferences(allow_yanked=True), - session=PipSession(), target_python=None, ) # Spot-check the default TargetPython object. @@ -721,11 +640,14 @@ def test_create__allow_yanked(self, allow_yanked): """ Test that the _allow_yanked attribute is set correctly. """ + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) selection_prefs = SelectionPreferences(allow_yanked=allow_yanked) finder = PackageFinder.create( - search_scope=SearchScope([], []), + link_collector=link_collector, selection_prefs=selection_prefs, - session=PipSession(), ) assert finder._allow_yanked == allow_yanked @@ -734,14 +656,17 @@ def test_create__ignore_requires_python(self, ignore_requires_python): """ Test that the _ignore_requires_python attribute is set correctly. """ + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) selection_prefs = SelectionPreferences( allow_yanked=True, ignore_requires_python=ignore_requires_python, ) finder = PackageFinder.create( - search_scope=SearchScope([], []), + link_collector=link_collector, selection_prefs=selection_prefs, - session=PipSession(), ) assert finder._ignore_requires_python == ignore_requires_python @@ -749,15 +674,18 @@ def test_create__format_control(self): """ Test that the format_control attribute is set correctly. """ + link_collector = LinkCollector( + session=PipSession(), + search_scope=SearchScope([], []), + ) format_control = FormatControl(set(), {':all:'}) selection_prefs = SelectionPreferences( allow_yanked=True, format_control=format_control, ) finder = PackageFinder.create( - search_scope=SearchScope([], []), + link_collector=link_collector, selection_prefs=selection_prefs, - session=PipSession(), ) actual_format_control = finder.format_control assert actual_format_control is format_control @@ -856,61 +784,6 @@ def test_make_candidate_evaluator( assert evaluator._supported_tags == [('py36', 'none', 'any')] -def test_group_locations__file_expand_dir(data): - """ - Test that a file:// dir gets listdir run with expand_dir - """ - files, urls = group_locations([data.find_links], expand_dir=True) - assert files and not urls, ( - "files and not urls should have been found at find-links url: %s" % - data.find_links - ) - - -def test_group_locations__file_not_find_link(data): - """ - Test that a file:// url dir that's not a find-link, doesn't get a listdir - run - """ - files, urls = group_locations([data.index_url("empty_with_pkg")]) - assert urls and not files, "urls, but not files should have been found" - - -def test_group_locations__non_existing_path(): - """ - Test that a non-existing path is ignored. - """ - files, urls = group_locations([os.path.join('this', 'doesnt', 'exist')]) - assert not urls and not files, "nothing should have been found" - - -@pytest.mark.parametrize( - ("html", "url", "expected"), - [ - (b"<html></html>", "https://example.com/", "https://example.com/"), - ( - b"<html><head>" - b"<base href=\"https://foo.example.com/\">" - b"</head></html>", - "https://example.com/", - "https://foo.example.com/", - ), - ( - b"<html><head>" - b"<base><base href=\"https://foo.example.com/\">" - b"</head></html>", - "https://example.com/", - "https://foo.example.com/", - ), - ], -) -def test_determine_base_url(html, url, expected): - document = html5lib.parse( - html, transport_encoding=None, namespaceHTMLElements=False, - ) - assert _determine_base_url(document, url) == expected - - @pytest.mark.parametrize( ("fragment", "canonical_name", "expected"), [ @@ -999,115 +872,3 @@ def test_find_name_version_sep_failure(fragment, canonical_name): def test_extract_version_from_fragment(fragment, canonical_name, expected): version = _extract_version_from_fragment(fragment, canonical_name) assert version == expected - - -def test_request_http_error(caplog): - caplog.set_level(logging.DEBUG) - link = Link('http://localhost') - session = Mock(PipSession) - session.get.return_value = resp = Mock() - resp.raise_for_status.side_effect = requests.HTTPError('Http error') - assert _get_html_page(link, session=session) is None - assert ( - 'Could not fetch URL http://localhost: Http error - skipping' - in caplog.text - ) - - -def test_request_retries(caplog): - caplog.set_level(logging.DEBUG) - link = Link('http://localhost') - session = Mock(PipSession) - session.get.side_effect = requests.exceptions.RetryError('Retry error') - assert _get_html_page(link, session=session) is None - assert ( - 'Could not fetch URL http://localhost: Retry error - skipping' - in caplog.text - ) - - -@pytest.mark.parametrize( - ("url", "clean_url"), - [ - # URL with hostname and port. Port separator should not be quoted. - ("https://localhost.localdomain:8181/path/with space/", - "https://localhost.localdomain:8181/path/with%20space/"), - # URL that is already properly quoted. The quoting `%` - # characters should not be quoted again. - ("https://localhost.localdomain:8181/path/with%20quoted%20space/", - "https://localhost.localdomain:8181/path/with%20quoted%20space/"), - # URL with IPv4 address and port. - ("https://127.0.0.1:8181/path/with space/", - "https://127.0.0.1:8181/path/with%20space/"), - # URL with IPv6 address and port. The `[]` brackets around the - # IPv6 address should not be quoted. - ("https://[fd00:0:0:236::100]:8181/path/with space/", - "https://[fd00:0:0:236::100]:8181/path/with%20space/"), - # URL with query. The leading `?` should not be quoted. - ("https://localhost.localdomain:8181/path/with/query?request=test", - "https://localhost.localdomain:8181/path/with/query?request=test"), - # URL with colon in the path portion. - ("https://localhost.localdomain:8181/path:/with:/colon", - "https://localhost.localdomain:8181/path%3A/with%3A/colon"), - # URL with something that looks like a drive letter, but is - # not. The `:` should be quoted. - ("https://localhost.localdomain/T:/path/", - "https://localhost.localdomain/T%3A/path/"), - # VCS URL containing revision string. - ("git+ssh://example.com/path to/repo.git@1.0#egg=my-package-1.0", - "git+ssh://example.com/path%20to/repo.git@1.0#egg=my-package-1.0"), - # URL with Windows drive letter. The `:` after the drive - # letter should not be quoted. The trailing `/` should be - # removed. - pytest.param( - "file:///T:/path/with spaces/", - "file:///T:/path/with%20spaces", - marks=pytest.mark.skipif("sys.platform != 'win32'"), - ), - # URL with Windows drive letter, running on non-windows - # platform. The `:` after the drive should be quoted. - pytest.param( - "file:///T:/path/with spaces/", - "file:///T%3A/path/with%20spaces/", - marks=pytest.mark.skipif("sys.platform == 'win32'"), - ), - ] -) -def test_clean_link(url, clean_url): - assert(_clean_link(url) == clean_url) - - -class TestHTMLPage: - - @pytest.mark.parametrize( - ('anchor_html, expected'), - [ - # Test not present. - ('<a href="/pkg1-1.0.tar.gz"></a>', None), - # Test present with no value. - ('<a href="/pkg2-1.0.tar.gz" data-yanked></a>', ''), - # Test the empty string. - ('<a href="/pkg3-1.0.tar.gz" data-yanked=""></a>', ''), - # Test a non-empty string. - ('<a href="/pkg4-1.0.tar.gz" data-yanked="error"></a>', 'error'), - # Test a value with an escaped character. - ('<a href="/pkg4-1.0.tar.gz" data-yanked="version < 1"></a>', - 'version < 1'), - # Test a yanked reason with a non-ascii character. - (u'<a href="/pkg-1.0.tar.gz" data-yanked="curlyquote \u2018"></a>', - u'curlyquote \u2018'), - ] - ) - def test_iter_links__yanked_reason(self, anchor_html, expected): - html = ( - # Mark this as a unicode string for Python 2 since anchor_html - # can contain non-ascii. - u'<html><head><meta charset="utf-8"><head>' - '<body>{}</body></html>' - ).format(anchor_html) - html_bytes = html.encode('utf-8') - page = HTMLPage(html_bytes, url='https://example.com/simple/') - links = list(page.iter_links()) - link, = links - actual = link.yanked_reason - assert actual == expected diff --git a/tests/unit/test_index_html_page.py b/tests/unit/test_index_html_page.py deleted file mode 100644 index ec2a3950e7a..00000000000 --- a/tests/unit/test_index_html_page.py +++ /dev/null @@ -1,194 +0,0 @@ -import logging - -import mock -import pytest -from pip._vendor.six.moves.urllib import request as urllib_request - -from pip._internal.download import PipSession -from pip._internal.index import ( - Link, - _get_html_page, - _get_html_response, - _NotHTML, - _NotHTTP, -) - - -@pytest.mark.parametrize( - "url", - [ - "ftp://python.org/python-3.7.1.zip", - "file:///opt/data/pip-18.0.tar.gz", - ], -) -def test_get_html_response_archive_to_naive_scheme(url): - """ - `_get_html_response()` should error on an archive-like URL if the scheme - does not allow "poking" without getting data. - """ - with pytest.raises(_NotHTTP): - _get_html_response(url, session=mock.Mock(PipSession)) - - -@pytest.mark.parametrize( - "url, content_type", - [ - ("http://python.org/python-3.7.1.zip", "application/zip"), - ("https://pypi.org/pip-18.0.tar.gz", "application/gzip"), - ], -) -def test_get_html_response_archive_to_http_scheme(url, content_type): - """ - `_get_html_response()` should send a HEAD request on an archive-like URL - if the scheme supports it, and raise `_NotHTML` if the response isn't HTML. - """ - session = mock.Mock(PipSession) - session.head.return_value = mock.Mock(**{ - "request.method": "HEAD", - "headers": {"Content-Type": content_type}, - }) - - with pytest.raises(_NotHTML) as ctx: - _get_html_response(url, session=session) - - session.assert_has_calls([ - mock.call.head(url, allow_redirects=True), - ]) - assert ctx.value.args == (content_type, "HEAD") - - -@pytest.mark.parametrize( - "url", - [ - "http://python.org/python-3.7.1.zip", - "https://pypi.org/pip-18.0.tar.gz", - ], -) -def test_get_html_response_archive_to_http_scheme_is_html(url): - """ - `_get_html_response()` should work with archive-like URLs if the HEAD - request is responded with text/html. - """ - session = mock.Mock(PipSession) - session.head.return_value = mock.Mock(**{ - "request.method": "HEAD", - "headers": {"Content-Type": "text/html"}, - }) - session.get.return_value = mock.Mock(headers={"Content-Type": "text/html"}) - - resp = _get_html_response(url, session=session) - - assert resp is not None - assert session.mock_calls == [ - mock.call.head(url, allow_redirects=True), - mock.call.head().raise_for_status(), - mock.call.get(url, headers={ - "Accept": "text/html", "Cache-Control": "max-age=0", - }), - mock.call.get().raise_for_status(), - ] - - -@pytest.mark.parametrize( - "url", - [ - "https://pypi.org/simple/pip", - "https://pypi.org/simple/pip/", - "https://python.org/sitemap.xml", - ], -) -def test_get_html_response_no_head(url): - """ - `_get_html_response()` shouldn't send a HEAD request if the URL does not - look like an archive, only the GET request that retrieves data. - """ - session = mock.Mock(PipSession) - - # Mock the headers dict to ensure it is accessed. - session.get.return_value = mock.Mock(headers=mock.Mock(**{ - "get.return_value": "text/html", - })) - - resp = _get_html_response(url, session=session) - - assert resp is not None - assert session.head.call_count == 0 - assert session.get.mock_calls == [ - mock.call(url, headers={ - "Accept": "text/html", "Cache-Control": "max-age=0", - }), - mock.call().raise_for_status(), - mock.call().headers.get("Content-Type", ""), - ] - - -def test_get_html_response_dont_log_clear_text_password(caplog): - """ - `_get_html_response()` should redact the password from the index URL - in its DEBUG log message. - """ - session = mock.Mock(PipSession) - - # Mock the headers dict to ensure it is accessed. - session.get.return_value = mock.Mock(headers=mock.Mock(**{ - "get.return_value": "text/html", - })) - - caplog.set_level(logging.DEBUG) - - resp = _get_html_response( - "https://user:my_password@example.com/simple/", session=session - ) - - assert resp is not None - - assert len(caplog.records) == 1 - record = caplog.records[0] - assert record.levelname == 'DEBUG' - assert record.message.splitlines() == [ - "Getting page https://user:****@example.com/simple/", - ] - - -@pytest.mark.parametrize( - "url, vcs_scheme", - [ - ("svn+http://pypi.org/something", "svn"), - ("git+https://github.com/pypa/pip.git", "git"), - ], -) -def test_get_html_page_invalid_scheme(caplog, url, vcs_scheme): - """`_get_html_page()` should error if an invalid scheme is given. - - Only file:, http:, https:, and ftp: are allowed. - """ - with caplog.at_level(logging.DEBUG): - page = _get_html_page(Link(url), session=mock.Mock(PipSession)) - - assert page is None - assert caplog.record_tuples == [ - ( - "pip._internal.index", - logging.DEBUG, - "Cannot look at {} URL {}".format(vcs_scheme, url), - ), - ] - - -def test_get_html_page_directory_append_index(tmpdir): - """`_get_html_page()` should append "index.html" to a directory URL. - """ - dirpath = tmpdir.mkdir("something") - dir_url = "file:///{}".format( - urllib_request.pathname2url(dirpath).lstrip("/"), - ) - - session = mock.Mock(PipSession) - with mock.patch("pip._internal.index._get_html_response") as mock_func: - _get_html_page(Link(dir_url), session=session) - assert mock_func.mock_calls == [ - mock.call( - "{}/index.html".format(dir_url.rstrip("/")), - session=session, - ), - ] diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index f048c548169..eef6ca373ef 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -22,6 +22,8 @@ from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import InstallRequirement, RequirementSet from pip._internal.req.constructors import ( + _get_url_from_path, + _looks_like_path, install_req_from_editable, install_req_from_line, install_req_from_req_string, @@ -343,6 +345,33 @@ def test_url_with_query(self): req = install_req_from_line(url + fragment) assert req.link.url == url + fragment, req.link + def test_pep440_wheel_link_requirement(self): + url = 'https://whatever.com/test-0.4-py2.py3-bogus-any.whl' + line = 'test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl' + req = install_req_from_line(line) + parts = str(req.req).split('@', 1) + assert len(parts) == 2 + assert parts[0].strip() == 'test' + assert parts[1].strip() == url + + def test_pep440_url_link_requirement(self): + url = 'git+http://foo.com@ref#egg=foo' + line = 'foo @ git+http://foo.com@ref#egg=foo' + req = install_req_from_line(line) + parts = str(req.req).split('@', 1) + assert len(parts) == 2 + assert parts[0].strip() == 'foo' + assert parts[1].strip() == url + + def test_url_with_authentication_link_requirement(self): + url = 'https://what@whatever.com/test-0.4-py2.py3-bogus-any.whl' + line = 'https://what@whatever.com/test-0.4-py2.py3-bogus-any.whl' + req = install_req_from_line(line) + assert req.link is not None + assert req.link.is_wheel + assert req.link.scheme == "https" + assert req.link.url == url + def test_unsupported_wheel_link_requirement_raises(self): reqset = RequirementSet() req = install_req_from_line( @@ -634,3 +663,95 @@ def test_mismatched_versions(caplog, tmpdir): 'Requested simplewheel==2.0, ' 'but installing version 1.0' ) + + +@pytest.mark.parametrize('args, expected', [ + # Test UNIX-like paths + (('/path/to/installable'), True), + # Test relative paths + (('./path/to/installable'), True), + # Test current path + (('.'), True), + # Test url paths + (('https://whatever.com/test-0.4-py2.py3-bogus-any.whl'), True), + # Test pep440 paths + (('test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl'), True), + # Test wheel + (('simple-0.1-py2.py3-none-any.whl'), False), +]) +def test_looks_like_path(args, expected): + assert _looks_like_path(args) == expected + + +@pytest.mark.skipif( + not sys.platform.startswith("win"), + reason='Test only available on Windows' +) +@pytest.mark.parametrize('args, expected', [ + # Test relative paths + (('.\\path\\to\\installable'), True), + (('relative\\path'), True), + # Test absolute paths + (('C:\\absolute\\path'), True), +]) +def test_looks_like_path_win(args, expected): + assert _looks_like_path(args) == expected + + +@pytest.mark.parametrize('args, mock_returns, expected', [ + # Test pep440 urls + (('/path/to/foo @ git+http://foo.com@ref#egg=foo', + 'foo @ git+http://foo.com@ref#egg=foo'), (False, False), None), + # Test pep440 urls without spaces + (('/path/to/foo@git+http://foo.com@ref#egg=foo', + 'foo @ git+http://foo.com@ref#egg=foo'), (False, False), None), + # Test pep440 wheel + (('/path/to/test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl', + 'test @ https://whatever.com/test-0.4-py2.py3-bogus-any.whl'), + (False, False), None), + # Test name is not a file + (('/path/to/simple==0.1', + 'simple==0.1'), + (False, False), None), +]) +@patch('pip._internal.req.req_install.os.path.isdir') +@patch('pip._internal.req.req_install.os.path.isfile') +def test_get_url_from_path( + isdir_mock, isfile_mock, args, mock_returns, expected +): + isdir_mock.return_value = mock_returns[0] + isfile_mock.return_value = mock_returns[1] + assert _get_url_from_path(*args) is expected + + +@patch('pip._internal.req.req_install.os.path.isdir') +@patch('pip._internal.req.req_install.os.path.isfile') +def test_get_url_from_path__archive_file(isdir_mock, isfile_mock): + isdir_mock.return_value = False + isfile_mock.return_value = True + name = 'simple-0.1-py2.py3-none-any.whl' + path = os.path.join('/path/to/' + name) + url = path_to_url(path) + assert _get_url_from_path(path, name) == url + + +@patch('pip._internal.req.req_install.os.path.isdir') +@patch('pip._internal.req.req_install.os.path.isfile') +def test_get_url_from_path__installable_dir(isdir_mock, isfile_mock): + isdir_mock.return_value = True + isfile_mock.return_value = True + name = 'some/setuptools/project' + path = os.path.join('/path/to/' + name) + url = path_to_url(path) + assert _get_url_from_path(path, name) == url + + +@patch('pip._internal.req.req_install.os.path.isdir') +def test_get_url_from_path__installable_error(isdir_mock): + isdir_mock.return_value = True + name = 'some/setuptools/project' + path = os.path.join('/path/to/' + name) + with pytest.raises(InstallationError) as e: + _get_url_from_path(path, name) + err_msg = e.value.args[0] + assert "Neither 'setup.py' nor 'pyproject.toml' found" in err_msg diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 81a94a715fc..8584c11301a 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -225,6 +225,13 @@ def test_yield_line_requirement(self): req = install_req_from_line(line, comes_from=comes_from) assert repr(list(process_line(line, filename, 1))[0]) == repr(req) + def test_yield_pep440_line_requirement(self): + line = 'SomeProject @ https://url/SomeProject-py2-py3-none-any.whl' + filename = 'filename' + comes_from = '-r %s (line %s)' % (filename, 1) + req = install_req_from_line(line, comes_from=comes_from) + assert repr(list(process_line(line, filename, 1))[0]) == repr(req) + def test_yield_line_constraint(self): line = 'SomeProject' filename = 'filename' diff --git a/tests/unit/test_req_install.py b/tests/unit/test_req_install.py index a0c7711dcae..a8eae8249bb 100644 --- a/tests/unit/test_req_install.py +++ b/tests/unit/test_req_install.py @@ -20,7 +20,7 @@ def test_tmp_build_directory(self): # Make sure we're handling it correctly with real path. requirement = InstallRequirement(None, None) tmp_dir = tempfile.mkdtemp('-build', 'pip-') - tmp_build_dir = requirement.build_location(tmp_dir) + tmp_build_dir = requirement.ensure_build_location(tmp_dir) assert ( os.path.dirname(tmp_build_dir) == os.path.realpath(os.path.dirname(tmp_dir)) diff --git a/tests/unit/test_req_uninstall.py b/tests/unit/test_req_uninstall.py index 69dbeebfe8d..d4d707e6042 100644 --- a/tests/unit/test_req_uninstall.py +++ b/tests/unit/test_req_uninstall.py @@ -183,7 +183,7 @@ def test_add_symlink(self, tmpdir, monkeypatch): def test_compact_shorter_path(self, monkeypatch): monkeypatch.setattr(pip._internal.req.req_uninstall, 'is_local', - lambda p: True) + mock_is_local) monkeypatch.setattr('os.path.exists', lambda p: True) # This deals with nt/posix path differences short_path = os.path.normcase(os.path.abspath( @@ -196,7 +196,7 @@ def test_compact_shorter_path(self, monkeypatch): @pytest.mark.skipif("sys.platform == 'win32'") def test_detect_symlink_dirs(self, monkeypatch, tmpdir): monkeypatch.setattr(pip._internal.req.req_uninstall, 'is_local', - lambda p: True) + mock_is_local) # construct 2 paths: # tmpdir/dir/file @@ -306,3 +306,67 @@ def test_rollback(self, tmpdir): for old_path, new_path in stashed_paths: assert os.path.exists(old_path) assert not os.path.exists(new_path) + + @pytest.mark.skipif("sys.platform == 'win32'") + def test_commit_symlinks(self, tmpdir): + adir = tmpdir / "dir" + adir.mkdir() + dirlink = tmpdir / "dirlink" + dirlink.symlink_to(adir) + afile = tmpdir / "file" + afile.write_text("...") + filelink = tmpdir / "filelink" + filelink.symlink_to(afile) + + pathset = StashedUninstallPathSet() + stashed_paths = [] + stashed_paths.append(pathset.stash(dirlink)) + stashed_paths.append(pathset.stash(filelink)) + for stashed_path in stashed_paths: + assert os.path.lexists(stashed_path) + assert not os.path.exists(dirlink) + assert not os.path.exists(filelink) + + pathset.commit() + + # stash removed, links removed + for stashed_path in stashed_paths: + assert not os.path.lexists(stashed_path) + assert not os.path.lexists(dirlink) and not os.path.isdir(dirlink) + assert not os.path.lexists(filelink) and not os.path.isfile(filelink) + + # link targets untouched + assert os.path.isdir(adir) + assert os.path.isfile(afile) + + @pytest.mark.skipif("sys.platform == 'win32'") + def test_rollback_symlinks(self, tmpdir): + adir = tmpdir / "dir" + adir.mkdir() + dirlink = tmpdir / "dirlink" + dirlink.symlink_to(adir) + afile = tmpdir / "file" + afile.write_text("...") + filelink = tmpdir / "filelink" + filelink.symlink_to(afile) + + pathset = StashedUninstallPathSet() + stashed_paths = [] + stashed_paths.append(pathset.stash(dirlink)) + stashed_paths.append(pathset.stash(filelink)) + for stashed_path in stashed_paths: + assert os.path.lexists(stashed_path) + assert not os.path.lexists(dirlink) + assert not os.path.lexists(filelink) + + pathset.rollback() + + # stash removed, links restored + for stashed_path in stashed_paths: + assert not os.path.lexists(stashed_path) + assert os.path.lexists(dirlink) and os.path.isdir(dirlink) + assert os.path.lexists(filelink) and os.path.isfile(filelink) + + # link targets untouched + assert os.path.isdir(adir) + assert os.path.isfile(afile) diff --git a/tests/unit/test_unit_outdated.py b/tests/unit/test_unit_outdated.py index 092e7499205..4dd5b16cb93 100644 --- a/tests/unit/test_unit_outdated.py +++ b/tests/unit/test_unit_outdated.py @@ -2,23 +2,96 @@ import json import os import sys -from contextlib import contextmanager import freezegun import pretend import pytest -from pip._vendor import lockfile, pkg_resources +from mock import patch +from pip._vendor import pkg_resources +from pip._internal.download import PipSession from pip._internal.index import InstallationCandidate from pip._internal.utils import outdated from pip._internal.utils.outdated import ( SelfCheckState, logger, + make_link_collector, pip_version_check, ) from tests.lib.path import Path +@pytest.mark.parametrize( + 'find_links, no_index, suppress_no_index, expected', [ + (['link1'], False, False, + (['link1'], ['default_url', 'url1', 'url2'])), + (['link1'], False, True, (['link1'], ['default_url', 'url1', 'url2'])), + (['link1'], True, False, (['link1'], [])), + # Passing suppress_no_index=True suppresses no_index=True. + (['link1'], True, True, (['link1'], ['default_url', 'url1', 'url2'])), + # Test options.find_links=False. + (False, False, False, ([], ['default_url', 'url1', 'url2'])), + ], +) +def test_make_link_collector( + find_links, no_index, suppress_no_index, expected, +): + """ + :param expected: the expected (find_links, index_urls) values. + """ + expected_find_links, expected_index_urls = expected + session = PipSession() + options = pretend.stub( + find_links=find_links, + index_url='default_url', + extra_index_urls=['url1', 'url2'], + no_index=no_index, + ) + link_collector = make_link_collector( + session, options=options, suppress_no_index=suppress_no_index, + ) + + assert link_collector.session is session + + search_scope = link_collector.search_scope + assert search_scope.find_links == expected_find_links + assert search_scope.index_urls == expected_index_urls + + +@patch('pip._internal.utils.misc.expanduser') +def test_make_link_collector__find_links_expansion(mock_expanduser, tmpdir): + """ + Test "~" expansion in --find-links paths. + """ + # This is a mock version of expanduser() that expands "~" to the tmpdir. + def expand_path(path): + if path.startswith('~/'): + path = os.path.join(tmpdir, path[2:]) + return path + + mock_expanduser.side_effect = expand_path + + session = PipSession() + options = pretend.stub( + find_links=['~/temp1', '~/temp2'], + index_url='default_url', + extra_index_urls=[], + no_index=False, + ) + # Only create temp2 and not temp1 to test that "~" expansion only occurs + # when the directory exists. + temp2_dir = os.path.join(tmpdir, 'temp2') + os.mkdir(temp2_dir) + + link_collector = make_link_collector(session, options=options) + + search_scope = link_collector.search_scope + # Only ~/temp2 gets expanded. Also, the path is normalized when expanded. + expected_temp2_dir = os.path.normcase(temp2_dir) + assert search_scope.find_links == ['~/temp1', expected_temp2_dir] + assert search_scope.index_urls == ['default_url'] + + class MockBestCandidateResult(object): def __init__(self, best): self.best_candidate = best @@ -162,49 +235,6 @@ def _get_statefile_path(cache_dir, key): ) -def test_self_check_state(monkeypatch, tmpdir): - CONTENT = '''{"key": "pip_prefix", "last_check": "1970-01-02T11:00:00Z", - "pypi_version": "1.0"}''' - fake_file = pretend.stub( - read=pretend.call_recorder(lambda: CONTENT), - write=pretend.call_recorder(lambda s: None), - ) - - @pretend.call_recorder - @contextmanager - def fake_open(filename, mode='r'): - yield fake_file - - monkeypatch.setattr(outdated, 'open', fake_open, raising=False) - - @pretend.call_recorder - @contextmanager - def fake_lock(filename): - yield - - monkeypatch.setattr(outdated, "check_path_owner", lambda p: True) - - monkeypatch.setattr(lockfile, 'LockFile', fake_lock) - - cache_dir = tmpdir / 'cache_dir' - key = 'pip_prefix' - monkeypatch.setattr(sys, 'prefix', key) - - state = SelfCheckState(cache_dir=cache_dir) - state.save('2.0', datetime.datetime.utcnow()) - - expected_path = _get_statefile_path(str(cache_dir), key) - assert fake_lock.calls == [pretend.call(expected_path)] - - assert fake_open.calls == [ - pretend.call(expected_path), - pretend.call(expected_path, 'w'), - ] - - # json.dumps will call this a number of times - assert len(fake_file.write.calls) - - def test_self_check_state_no_cache_dir(): state = SelfCheckState(cache_dir=False) assert state.state == {} diff --git a/tests/unit/test_urls.py b/tests/unit/test_urls.py new file mode 100644 index 00000000000..68d544072f8 --- /dev/null +++ b/tests/unit/test_urls.py @@ -0,0 +1,48 @@ +import sys + +import pytest + +from pip._internal.utils.misc import path_to_url +from pip._internal.utils.urls import get_url_scheme, url_to_path + + +@pytest.mark.parametrize("url,expected", [ + ('http://localhost:8080/', 'http'), + ('file:c:/path/to/file', 'file'), + ('file:/dev/null', 'file'), + ('', None), +]) +def test_get_url_scheme(url, expected): + assert get_url_scheme(url) == expected + + +@pytest.mark.parametrize("url,win_expected,non_win_expected", [ + ('file:tmp', 'tmp', 'tmp'), + ('file:c:/path/to/file', r'C:\path\to\file', 'c:/path/to/file'), + ('file:/path/to/file', r'\path\to\file', '/path/to/file'), + ('file://localhost/tmp/file', r'\tmp\file', '/tmp/file'), + ('file://localhost/c:/tmp/file', r'C:\tmp\file', '/c:/tmp/file'), + ('file://somehost/tmp/file', r'\\somehost\tmp\file', None), + ('file:///tmp/file', r'\tmp\file', '/tmp/file'), + ('file:///c:/tmp/file', r'C:\tmp\file', '/c:/tmp/file'), +]) +def test_url_to_path(url, win_expected, non_win_expected): + if sys.platform == 'win32': + expected_path = win_expected + else: + expected_path = non_win_expected + + if expected_path is None: + with pytest.raises(ValueError): + url_to_path(url) + else: + assert url_to_path(url) == expected_path + + +@pytest.mark.skipif("sys.platform != 'win32'") +def test_url_to_path_path_to_url_symmetry_win(): + path = r'C:\tmp\file' + assert url_to_path(path_to_url(path)) == path + + unc_path = r'\\unc\share\path' + assert url_to_path(path_to_url(unc_path)) == unc_path diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 6c1ad16f807..c34dea9165f 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -59,10 +59,9 @@ redact_netloc, remove_auth_from_url, rmtree, + rmtree_errorhandler, split_auth_from_netloc, split_auth_netloc_from_url, - untar_file, - unzip_file, ) from pip._internal.utils.setuptools_build import make_setuptools_shim_args from pip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory @@ -302,88 +301,60 @@ def test_freeze_excludes(self, mock_dist_is_editable, assert len(dists) == 0 -class TestUnpackArchives(object): +def test_rmtree_errorhandler_nonexistent_directory(tmpdir): """ - test_tar.tgz/test_tar.zip have content as follows engineered to confirm 3 - things: - 1) confirm that reg files, dirs, and symlinks get unpacked - 2) permissions are not preserved (and go by the 022 umask) - 3) reg files with *any* execute perms, get chmod +x - - file.txt 600 regular file - symlink.txt 777 symlink to file.txt - script_owner.sh 700 script where owner can execute - script_group.sh 610 script where group can execute - script_world.sh 601 script where world can execute - dir 744 directory - dir/dirfile 622 regular file - 4) the file contents are extracted correctly (though the content of - each file isn't currently unique) + Test rmtree_errorhandler ignores the given non-existing directory. + """ + nonexistent_path = str(tmpdir / 'foo') + mock_func = Mock() + rmtree_errorhandler(mock_func, nonexistent_path, None) + mock_func.assert_not_called() + +def test_rmtree_errorhandler_readonly_directory(tmpdir): + """ + Test rmtree_errorhandler makes the given read-only directory writable. """ + # Create read only directory + path = str((tmpdir / 'subdir').mkdir()) + os.chmod(path, stat.S_IREAD) - def setup(self): - self.tempdir = tempfile.mkdtemp() - self.old_mask = os.umask(0o022) - self.symlink_expected_mode = None + # Make sure mock_func is called with the given path + mock_func = Mock() + rmtree_errorhandler(mock_func, path, None) + mock_func.assert_called_with(path) - def teardown(self): - os.umask(self.old_mask) - shutil.rmtree(self.tempdir, ignore_errors=True) - - def mode(self, path): - return stat.S_IMODE(os.stat(path).st_mode) - - def confirm_files(self): - # expectations based on 022 umask set above and the unpack logic that - # sets execute permissions, not preservation - for fname, expected_mode, test, expected_contents in [ - ('file.txt', 0o644, os.path.isfile, b'file\n'), - # We don't test the "symlink.txt" contents for now. - ('symlink.txt', 0o644, os.path.isfile, None), - ('script_owner.sh', 0o755, os.path.isfile, b'file\n'), - ('script_group.sh', 0o755, os.path.isfile, b'file\n'), - ('script_world.sh', 0o755, os.path.isfile, b'file\n'), - ('dir', 0o755, os.path.isdir, None), - (os.path.join('dir', 'dirfile'), 0o644, os.path.isfile, b''), - ]: - path = os.path.join(self.tempdir, fname) - if path.endswith('symlink.txt') and sys.platform == 'win32': - # no symlinks created on windows - continue - assert test(path), path - if expected_contents is not None: - with open(path, mode='rb') as f: - contents = f.read() - assert contents == expected_contents, 'fname: {}'.format(fname) - if sys.platform == 'win32': - # the permissions tests below don't apply in windows - # due to os.chmod being a noop - continue - mode = self.mode(path) - assert mode == expected_mode, ( - "mode: %s, expected mode: %s" % (mode, expected_mode) - ) + # Make sure the path is now writable + assert os.stat(path).st_mode & stat.S_IWRITE - def test_unpack_tgz(self, data): - """ - Test unpacking a *.tgz, and setting execute permissions - """ - test_file = data.packages.joinpath("test_tar.tgz") - untar_file(test_file, self.tempdir) - self.confirm_files() - # Check the timestamp of an extracted file - file_txt_path = os.path.join(self.tempdir, 'file.txt') - mtime = time.gmtime(os.stat(file_txt_path).st_mtime) - assert mtime[0:6] == (2013, 8, 16, 5, 13, 37), mtime - - def test_unpack_zip(self, data): - """ - Test unpacking a *.zip, and setting execute permissions - """ - test_file = data.packages.joinpath("test_zip.zip") - unzip_file(test_file, self.tempdir) - self.confirm_files() + +def test_rmtree_errorhandler_reraises_error(tmpdir): + """ + Test rmtree_errorhandler reraises an exception + by the given unreadable directory. + """ + # Create directory without read permission + path = str((tmpdir / 'subdir').mkdir()) + os.chmod(path, stat.S_IWRITE) + + mock_func = Mock() + + try: + raise RuntimeError('test message') + except RuntimeError: + # Make sure the handler reraises an exception + with pytest.raises(RuntimeError, match='test message'): + rmtree_errorhandler(mock_func, path, None) + + mock_func.assert_not_called() + + +def test_rmtree_skips_nonexistent_directory(): + """ + Test wrapped rmtree doesn't raise an error + by the given nonexistent directory. + """ + rmtree.__wrapped__('nonexistent-subdir') class Failer: diff --git a/tests/unit/test_utils_unpacking.py b/tests/unit/test_utils_unpacking.py new file mode 100644 index 00000000000..96fcb99569f --- /dev/null +++ b/tests/unit/test_utils_unpacking.py @@ -0,0 +1,92 @@ +import os +import shutil +import stat +import sys +import tempfile +import time + +from pip._internal.utils.unpacking import untar_file, unzip_file + + +class TestUnpackArchives(object): + """ + test_tar.tgz/test_tar.zip have content as follows engineered to confirm 3 + things: + 1) confirm that reg files, dirs, and symlinks get unpacked + 2) permissions are not preserved (and go by the 022 umask) + 3) reg files with *any* execute perms, get chmod +x + + file.txt 600 regular file + symlink.txt 777 symlink to file.txt + script_owner.sh 700 script where owner can execute + script_group.sh 610 script where group can execute + script_world.sh 601 script where world can execute + dir 744 directory + dir/dirfile 622 regular file + 4) the file contents are extracted correctly (though the content of + each file isn't currently unique) + + """ + + def setup(self): + self.tempdir = tempfile.mkdtemp() + self.old_mask = os.umask(0o022) + self.symlink_expected_mode = None + + def teardown(self): + os.umask(self.old_mask) + shutil.rmtree(self.tempdir, ignore_errors=True) + + def mode(self, path): + return stat.S_IMODE(os.stat(path).st_mode) + + def confirm_files(self): + # expectations based on 022 umask set above and the unpack logic that + # sets execute permissions, not preservation + for fname, expected_mode, test, expected_contents in [ + ('file.txt', 0o644, os.path.isfile, b'file\n'), + # We don't test the "symlink.txt" contents for now. + ('symlink.txt', 0o644, os.path.isfile, None), + ('script_owner.sh', 0o755, os.path.isfile, b'file\n'), + ('script_group.sh', 0o755, os.path.isfile, b'file\n'), + ('script_world.sh', 0o755, os.path.isfile, b'file\n'), + ('dir', 0o755, os.path.isdir, None), + (os.path.join('dir', 'dirfile'), 0o644, os.path.isfile, b''), + ]: + path = os.path.join(self.tempdir, fname) + if path.endswith('symlink.txt') and sys.platform == 'win32': + # no symlinks created on windows + continue + assert test(path), path + if expected_contents is not None: + with open(path, mode='rb') as f: + contents = f.read() + assert contents == expected_contents, 'fname: {}'.format(fname) + if sys.platform == 'win32': + # the permissions tests below don't apply in windows + # due to os.chmod being a noop + continue + mode = self.mode(path) + assert mode == expected_mode, ( + "mode: %s, expected mode: %s" % (mode, expected_mode) + ) + + def test_unpack_tgz(self, data): + """ + Test unpacking a *.tgz, and setting execute permissions + """ + test_file = data.packages.joinpath("test_tar.tgz") + untar_file(test_file, self.tempdir) + self.confirm_files() + # Check the timestamp of an extracted file + file_txt_path = os.path.join(self.tempdir, 'file.txt') + mtime = time.gmtime(os.stat(file_txt_path).st_mtime) + assert mtime[0:6] == (2013, 8, 16, 5, 13, 37), mtime + + def test_unpack_zip(self, data): + """ + Test unpacking a *.zip, and setting execute permissions + """ + test_file = data.packages.joinpath("test_zip.zip") + unzip_file(test_file, self.tempdir) + self.confirm_files() diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index c99b8f5d247..3cf47d51de4 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -13,7 +13,7 @@ from pip._internal.models.link import Link from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.compat import WINDOWS -from pip._internal.utils.misc import unpack_file +from pip._internal.utils.unpacking import unpack_file from pip._internal.wheel import ( MissingCallableSuffix, _raise_for_invalid_entrypoint, @@ -218,6 +218,7 @@ def test_get_legacy_build_wheel_path(caplog): def test_get_legacy_build_wheel_path__no_names(caplog): + caplog.set_level(logging.INFO) actual = call_get_legacy_build_wheel_path(caplog, names=[]) assert actual is None assert len(caplog.records) == 1 @@ -231,6 +232,7 @@ def test_get_legacy_build_wheel_path__no_names(caplog): def test_get_legacy_build_wheel_path__multiple_names(caplog): + caplog.set_level(logging.INFO) # Deliberately pass the names in non-sorted order. actual = call_get_legacy_build_wheel_path( caplog, names=['name2', 'name1'], @@ -368,9 +370,9 @@ def test_wheel_version(tmpdir, data): future_version = (1, 9) unpack_file(data.packages.joinpath(future_wheel), - tmpdir + 'future', None, None) + tmpdir + 'future', None) unpack_file(data.packages.joinpath(broken_wheel), - tmpdir + 'broken', None, None) + tmpdir + 'broken', None) assert wheel.wheel_version(tmpdir + 'future') == future_version assert not wheel.wheel_version(tmpdir + 'broken') @@ -588,20 +590,11 @@ def test_support_index_min__none_supported(self): with pytest.raises(ValueError): w.support_index_min(tags=[]) - def test_unpack_wheel_no_flatten(self): - from pip._internal.utils import misc as utils - from tempfile import mkdtemp - from shutil import rmtree - + def test_unpack_wheel_no_flatten(self, tmpdir): filepath = os.path.join(DATA_DIR, 'packages', 'meta-1.0-py2.py3-none-any.whl') - try: - tmpdir = mkdtemp() - utils.unpack_file(filepath, tmpdir, 'application/zip', None) - assert os.path.isdir(os.path.join(tmpdir, 'meta-1.0.dist-info')) - finally: - rmtree(tmpdir) - pass + unpack_file(filepath, tmpdir, 'application/zip') + assert os.path.isdir(os.path.join(tmpdir, 'meta-1.0.dist-info')) def test_purelib_platlib(self, data): """ @@ -640,7 +633,7 @@ def prep(self, data, tmpdir): self.req = Requirement('sample') self.src = os.path.join(tmpdir, 'src') self.dest = os.path.join(tmpdir, 'dest') - unpack_file(self.wheelpath, self.src, None, None) + unpack_file(self.wheelpath, self.src, None) self.scheme = { 'scripts': os.path.join(self.dest, 'bin'), 'purelib': os.path.join(self.dest, 'lib'),