wheel support (and transitively PEP-518), and pipe display failed com…

…mands

PEP-518 support: provide a tox configuration flag ``build`` which can be
either ``sdist`` or ``wheel``. For ``sdist`` (default) we build the
package as before by using ``python setup.py sdist``. However, when
``wheel`` is enabled now we'll use ``pip wheel`` to build it, and we'll
also install wheels in these case into the environments. Note: ``pip``
10 supports specifying project dependencies (such as ``setuptools-scm``,
or a given ``setuptools`` version) via ``pyproject.toml``. Once ``pip``
supports building ``sdist`` to we'll migrate over the ``sdist`` build
too.

While running tox invokes various commands (such as building the
package, pip installing dependencies and so on), these were printed in
case they failed as Python arrays. Changed the representation to a shell
command, allowing the users to quickly replicate/debug the failure on
their own.

changelog/850.feature.rst

@@ -0,0 +1 @@
+PEP-518 support: provide a tox configuration flag ``build`` which can be either ``sdist`` or ``wheel``. For ``sdist`` (default) we build the package as before by using ``python setup.py sdist``. However, when ``wheel`` is enabled now we'll use ``pip wheel`` to build it, and we'll also install wheels in these case into the environments. Note: ``pip`` 10 supports specifying project dependencies (such as ``setuptools-scm``, or a given ``setuptools`` version) via ``pyproject.toml``. Once ``pip`` supports building ``sdist`` to we'll migrate over the ``sdist`` build too. @gaborbernat

changelog/851.feature.rst

@@ -0,0 +1 @@
+While running tox invokes various commands (such as building the package, pip installing dependencies and so on), these were printed in case they failed as Python arrays. Changed the representation to a shell command, allowing the users to quickly replicate/debug the failure on their own. @gaborbernat

src/tox/_pytestplugin.py

@@ -1,6 +1,7 @@
 from __future__ import print_function, unicode_literals
 
 import os
+import re
 import textwrap
 import time
 from fnmatch import fnmatch
@@ -61,15 +62,12 @@ def create_new_config_file_(args, source=None, plugins=()):
 
 
 @pytest.fixture
-def cmd(request, capfd, monkeypatch):
+def cmd(request, capfd):
     if request.config.option.no_network:
         pytest.skip("--no-network was specified, test cannot run")
     request.addfinalizer(py.path.local().chdir)
 
     def run(*argv):
-        key = str(b"PYTHONPATH")
-        python_paths = (i for i in (str(os.getcwd()), os.getenv(key)) if i)
-        monkeypatch.setenv(key, os.pathsep.join(python_paths))
         with RunResult(capfd, argv) as result:
             try:
                 main([str(x) for x in argv])
@@ -111,6 +109,10 @@ def __repr__(self):
             self.ret, " ".join(str(i) for i in self.args), self.out, self.err
         )
 
+    @property
+    def python_hash_seed(self):
+        return next(re.finditer(r"PYTHONHASHSEED='([0-9]+)'", self.out)).group(1)
+
 
 class ReportExpectMock:
     def __init__(self, session):

src/tox/config.py

@@ -943,6 +943,7 @@ def __init__(self, config, inipath):
             for name in config.indexserver:
                 config.indexserver[name] = IndexServerConfig(name, override)
 
+        config.build = reader.getstring("build", default="sdist")
         reader.addsubstitutions(toxworkdir=config.toxworkdir)
         config.distdir = reader.getpath("distdir", "{toxworkdir}/dist")
         reader.addsubstitutions(distdir=config.distdir)

src/tox/session.py

@@ -7,6 +7,7 @@
 from __future__ import print_function
 
 import os
+import pipes
 import re
 import shutil
 import subprocess
@@ -136,9 +137,16 @@ def _initlogpath(self, actionid):
     def popen(self, args, cwd=None, env=None, redirect=True, returnout=False, ignore_ret=False):
         stdout = outpath = None
         resultjson = self.session.config.option.resultjson
+
+        cmd_args = [str(x) for x in args]
+        cmd_args_shell = " ".join(pipes.quote(i) for i in cmd_args)
         if resultjson or redirect:
             fout = self._initlogpath(self.id)
-            fout.write("actionid: {}\nmsg: {}\ncmdargs: {!r}\n\n".format(self.id, self.msg, args))
+            fout.write(
+                "actionid: {}\nmsg: {}\ncmdargs: {!r}\n\n".format(
+                    self.id, self.msg, cmd_args_shell
+                )
+            )
             fout.flush()
             outpath = py.path.local(fout.name)
             fin = outpath.open("rb")
@@ -153,11 +161,13 @@ def popen(self, args, cwd=None, env=None, redirect=True, returnout=False, ignore
             popen = self._popen(args, cwd, env=env, stdout=stdout, stderr=subprocess.STDOUT)
         except OSError as e:
             self.report.error(
-                "invocation failed (errno {:d}), args: {}, cwd: {}".format(e.errno, args, cwd)
+                "invocation failed (errno {:d}), args: {}, cwd: {}".format(
+                    e.errno, cmd_args_shell, cwd
+                )
             )
             raise
         popen.outpath = outpath
-        popen.args = [str(x) for x in args]
+        popen.args = cmd_args
         popen.cwd = cwd
         popen.action = self
         self._popenlist.append(popen)
@@ -431,7 +441,7 @@ def _copyfiles(self, srcdir, pathlist, destdir):
             target.dirpath().ensure(dir=1)
             src.copy(target)
 
-    def _makesdist(self):
+    def _get_package(self):
         setup = self.config.setupdir.join("setup.py")
         if not setup.check():
             self.report.error(
@@ -446,19 +456,8 @@ def _makesdist(self):
             )
             raise SystemExit(1)
         with self.newaction(None, "packaging") as action:
-            action.setactivity("sdist-make", setup)
-            self.make_emptydir(self.config.distdir)
-            action.popen(
-                [
-                    sys.executable,
-                    setup,
-                    "sdist",
-                    "--formats=zip",
-                    "--dist-dir",
-                    self.config.distdir,
-                ],
-                cwd=self.config.setupdir,
-            )
+            action.setactivity("{}-make".format(self.config.build), setup)
+            self._make_package(action, setup)
             try:
                 return self.config.distdir.listdir()[0]
             except py.error.ENOENT:
@@ -478,6 +477,37 @@ def _makesdist(self):
                 )
                 raise SystemExit(1)
 
+    def _make_package(self, action, setup):
+        self.make_emptydir(self.config.distdir)
+        if self.config.build == "sdist":
+            action.popen(
+                [
+                    sys.executable,
+                    setup,
+                    "sdist",
+                    "--formats=zip",
+                    "--dist-dir",
+                    self.config.distdir,
+                ],
+                cwd=self.config.setupdir,
+            )
+        elif self.config.build == "wheel":
+            action.popen(
+                [
+                    sys.executable,
+                    "-m",
+                    "pip",
+                    "wheel",
+                    "-w",
+                    self.config.distdir,  # target folder
+                    "--no-deps",  # don't build wheels for package dependencies
+                    self.config.setupdir,  # what to build - ourselves
+                ],
+                cwd=self.config.setupdir,
+            )
+        else:
+            raise RuntimeError("invalid build type {}".format(self.config.build))
+
     def make_emptydir(self, path):
         if path.check():
             self.report.info("  removing {}".format(path))
@@ -576,23 +606,23 @@ def get_installpkg_path(self):
             if not path:
                 path = self.config.sdistsrc
             path = self._resolve_pkg(path)
-            self.report.info("using package {!r}, skipping 'sdist' activity ".format(str(path)))
+            self.report.info(
+                "using package {!r}, skipping '{}' activity ".format(str(path), self.config.build)
+            )
         else:
             try:
-                path = self._makesdist()
+                path = self._get_package()
             except tox.exception.InvocationError:
                 v = sys.exc_info()[1]
                 self.report.error("FAIL could not package project - v = {!r}".format(v))
                 return
             sdistfile = self.config.distshare.join(path.basename)
             if sdistfile != path:
-                self.report.info("copying new sdistfile to {!r}".format(str(sdistfile)))
+                self.report.info("copying new package to {!r}".format(str(sdistfile)))
                 try:
                     sdistfile.dirpath().ensure(dir=1)
                 except py.error.Error:
-                    self.report.warning(
-                        "could not copy distfile to {}".format(sdistfile.dirpath())
-                    )
+                    self.report.warning("could not copy package to {}".format(sdistfile.dirpath()))
                 else:
                     path.copy(sdistfile)
         return path

tests/test_package_build.py

@@ -0,0 +1,155 @@
+import os
+import subprocess
+import textwrap
+
+import pytest
+
+from tox.config import parseconfig
+from tox.session import Session
+
+
+def test_make_sdist(initproj):
+    initproj(
+        "example123-0.5",
+        filedefs={
+            "tests": {"test_hello.py": "def test_hello(): pass"},
+            "tox.ini": """
+        """,
+        },
+    )
+    config = parseconfig([])
+    session = Session(config)
+    sdist = session.get_installpkg_path()
+    assert sdist.check()
+    assert sdist.ext == ".zip"
+    assert sdist == config.distdir.join(sdist.basename)
+    sdist_2 = session.get_installpkg_path()
+    assert sdist_2 == sdist
+    sdist.write("hello")
+    assert sdist.stat().size < 10
+    sdist_new = Session(config).get_installpkg_path()
+    assert sdist_new == sdist
+    assert sdist_new.stat().size > 10
+
+
+def test_make_sdist_distshare(tmpdir, initproj):
+    distshare = tmpdir.join("distshare")
+    initproj(
+        "example123-0.6",
+        filedefs={
+            "tests": {"test_hello.py": "def test_hello(): pass"},
+            "tox.ini": """
+        [tox]
+        distshare={}
+        """.format(
+                distshare
+            ),
+        },
+    )
+    config = parseconfig([])
+    session = Session(config)
+    sdist = session.get_installpkg_path()
+    assert sdist.check()
+    assert sdist.ext == ".zip"
+    assert sdist == config.distdir.join(sdist.basename)
+    sdist_share = config.distshare.join(sdist.basename)
+    assert sdist_share.check()
+    assert sdist_share.read("rb") == sdist.read("rb"), (sdist_share, sdist)
+
+
+def test_sdist_latest(tmpdir, newconfig):
+    distshare = tmpdir.join("distshare")
+    config = newconfig(
+        [],
+        """
+            [tox]
+            distshare={}
+            sdistsrc={{distshare}}/pkg123-*
+    """.format(
+            distshare
+        ),
+    )
+    p = distshare.ensure("pkg123-1.4.5.zip")
+    distshare.ensure("pkg123-1.4.5a1.zip")
+    session = Session(config)
+    sdist_path = session.get_installpkg_path()
+    assert sdist_path == p
+
+
+def test_installpkg(tmpdir, newconfig):
+    p = tmpdir.ensure("pkg123-1.0.zip")
+    config = newconfig(["--installpkg={}".format(p)], "")
+    session = Session(config)
+    sdist_path = session.get_installpkg_path()
+    assert sdist_path == p
+
+
+@pytest.mark.integration
+@pytest.mark.pip
+@pytest.mark.git
+def test_pyproject_toml_with_setuptools_scm(initproj, cmd):
+    initproj(
+        "demo",
+        filedefs={
+            "demo": {
+                "__init__.py": """
+                from pkg_resources import get_distribution
+                __version__ = get_distribution(__name__).version
+                """
+            },
+            "setup.py": """
+               from setuptools import setup, find_packages
+               setup(
+                   name='demo',
+                   use_scm_version=True,
+                   license='MIT',
+                   platforms=['unix', 'win32'],
+                   packages=find_packages('demo'),
+                   package_dir={'':'demo'},
+               )
+               """,
+            "setup.cfg": """
+            [bdist_wheel]
+            universal = 1
+                        """,
+            "pyproject.toml": """
+            [build-system]
+            requires = ["setuptools >= 35.0.2", "setuptools_scm >= 2.0.0, <3", "wheel >= 0.29.0"]
+            """,
+            "tox.ini": """
+                [tox]
+                build = wheel
+                envlist = py
+
+                [testenv]
+                passenv = PYTHONPATH
+                commands = python -c 'import demo; print(demo.__version__)'
+            """,
+        },
+    )
+    env = os.environ.copy()
+    env["GIT_COMMITTER_NAME"] = "committer joe"
+    env["GIT_AUTHOR_NAME"] = "author joe"
+    env["EMAIL"] = "[email protected]"
+    subprocess.check_call(["git", "init"], env=env)
+    subprocess.check_call(["git", "add", "."], env=env)
+    subprocess.check_call(["git", "commit", "-m", "first commit"], env=env)
+    subprocess.check_call(["git", "tag", "0.1"], env=env)
+
+    result = cmd()
+    expected = textwrap.dedent(
+        """    GLOB wheel-make: {0}/setup.py
+    py create: {0}/.tox/py
+    py inst: {0}/.tox/dist/demo-0.1-py2.py3-none-any.whl
+    py installed: demo==0.1
+    py runtests: PYTHONHASHSEED='{1}'
+    py runtests: commands[0] | python -c 'import demo; print(demo.__version__)'
+    0.1
+    ___________________________________ summary ____________________________________
+      py: commands succeeded
+      congratulations :)
+    """.format(
+            os.getcwd(), result.python_hash_seed
+        )
+    )
+    assert result.out == expected

tests/test_venv.py

@@ -257,7 +257,7 @@ def test_install_recreate(newmocksession, tmpdir):
     mocksession.report.expect("verbosity0", "*recreate*")
 
 
-def test_install_sdist_extras(newmocksession):
+def test_install_wheel_extras(newmocksession):
     mocksession = newmocksession(
         [],
         """
@@ -273,8 +273,8 @@ def test_install_sdist_extras(newmocksession):
     assert len(pcalls) == 1
     pcalls[:] = []
 
-    venv.installpkg("distfile.tar.gz", action=action)
-    assert "distfile.tar.gz[testing,development]" in pcalls[-1].args
+    venv.installpkg("distfile.whl", action=action)
+    assert "distfile.whl[testing,development]" in pcalls[-1].args
 
 
 def test_develop_extras(newmocksession, tmpdir):