Issue #3316: Include all deps in 'pipenv lock -r --dev'

* Implements PEEP-006
* `pipenv lock -r --dev` is now consistent with other commands
  and the CLI help output, and includes both default and dev
  dependencies in the result
* New `--dev-only` option allows requesting the previous behaviour
  (which was specifically designed to support the traditional
  `requirements.txt`/`dev-requirements.txt` split)

docs/advanced.rst

@@ -149,7 +149,9 @@ Anaconda uses Conda to manage packages. To reuse Conda–installed Python packag
 ☤ Generating a ``requirements.txt``
 -----------------------------------
 
-You can convert a ``Pipfile`` and ``Pipfile.lock`` into a ``requirements.txt`` file very easily, and get all the benefits of extras and other goodies we have included.
+You can convert a ``Pipfile`` and ``Pipfile.lock`` into a ``requirements.txt``
+file very easily, and get all the benefits of extras and other goodies we have
+included.
 
 Let's take this ``Pipfile``::
 
@@ -160,7 +162,10 @@ Let's take this ``Pipfile``::
     [packages]
     requests = {version="*"}
 
-And generate a ``requirements.txt`` out of it::
+    [dev-packages]
+    pytest = {version="*"}
+
+And generate a set of requirements out of it with only the production dependencies::
 
     $ pipenv lock -r
     chardet==3.0.4
@@ -169,22 +174,41 @@ And generate a ``requirements.txt`` out of it::
     idna==2.6
     urllib3==1.22
 
-If you wish to generate a ``requirements.txt`` with only the development requirements you can do that too!  Let's take the following ``Pipfile``::
+As with other commands, passing ``--dev`` will include both the production and
+development dependencies::
 
-    [[source]]
-    url = "https://pypi.python.org/simple"
-    verify_ssl = true
+    $ pipenv lock -r --dev
+    chardet==3.0.4
+    requests==2.18.4
+    certifi==2017.7.27.1
+    idna==2.6
+    urllib3==1.22
+    py==1.4.34
+    pytest==3.2.3
 
-    [dev-packages]
-    pytest = {version="*"}
+Finally, if you wish to generate a requirements file with only the
+development requirements you can do that too, using the ``--dev-only``
+flag::
 
-And generate a ``requirements.txt`` out of it::
+    $ pipenv lock -r --dev-only
+    py==1.4.34
+    pytest==3.2.3
 
-    $ pipenv lock -r --dev
+The locked requirements are written to stdout, with shell output redirection
+used to write them to a file::
+
+    $ pipenv lock -r > requirements.txt
+    $ pipenv lock -r --dev-only > dev-requirements.txt
+    $ cat requirements.txt
+    chardet==3.0.4
+    requests==2.18.4
+    certifi==2017.7.27.1
+    idna==2.6
+    urllib3==1.22
+    $ cat dev-requirements.txt
     py==1.4.34
     pytest==3.2.3
 
-Very fancy.
 
 ☤ Detection of Security Vulnerabilities
 ---------------------------------------

news/3316.feature.rst

@@ -0,0 +1,5 @@
+For consistency with other commands and the ``-dev`` option
+description, ``pipenv lock --requirements --dev`` now emits
+both default and development dependencies.
+A new ``--dev-only`` has been added to request to previous
+behaviour (e.g. to generate a ``dev-requirements.txt`` file).

pipenv/cli/command.py

@@ -237,7 +237,7 @@ def install(
         lock=not state.installstate.skip_lock,
         ignore_pipfile=state.installstate.ignore_pipfile,
         skip_lock=state.installstate.skip_lock,
-        requirements=state.installstate.requirementstxt,
+        requirementstxt=state.installstate.requirementstxt,
         sequential=state.installstate.sequential,
         pre=state.installstate.pre,
         code=state.installstate.code,
@@ -317,13 +317,24 @@ def lock(
         three=state.three, python=state.python, pypi_mirror=state.pypi_mirror,
         warn=(not state.quiet), site_packages=state.site_packages,
     )
-    if state.installstate.requirementstxt:
+    if state.lockoptions.emit_requirements:
+        # Setting "requirements=True" means do_init() just emits the
+        # install requirements file to stdout, it doesn't install anything
+        if state.installstate.dev:
+            pass # TODO: Emit behaviour change warning as per PEEP 006
         do_init(
             dev=state.installstate.dev,
-            requirements=state.installstate.requirementstxt,
+            dev_only=state.lockoptions.dev_only,
+            emit_requirements=state.lockoptions.emit_requirements,
             pypi_mirror=state.pypi_mirror,
             pre=state.installstate.pre,
         )
+    elif state.lockoptions.dev_only:
+        raise exceptions.PipenvOptionsError(
+            "--dev-only",
+            "--dev-only is only permitted in combination with --requirements. "
+            "Aborting."
+        )
     do_lock(
         ctx=ctx,
         clear=state.clear,

pipenv/cli/options.py

@@ -65,6 +65,7 @@ def __init__(self):
         self.clear = False
         self.system = False
         self.installstate = InstallState()
+        self.lockoptions = LockOptions()
 
 
 class InstallState(object):
@@ -82,6 +83,10 @@ def __init__(self):
         self.packages = []
         self.editables = []
 
+class LockOptions(object):
+    def __init__(self):
+        self.dev_only = False
+        self.emit_requirements = False
 
 pass_state = make_pass_decorator(State, ensure=True)
 
@@ -300,15 +305,23 @@ def callback(ctx, param, value):
                   help="Import a requirements.txt file.", callback=callback)(f)
 
 
-def requirements_flag(f):
+def emit_requirements_flag(f):
     def callback(ctx, param, value):
         state = ctx.ensure_object(State)
         if value:
-            state.installstate.requirementstxt = value
+            state.lockoptions.emit_requirements = value
         return value
     return option("--requirements", "-r", default=False, is_flag=True, expose_value=False,
                   help="Generate output in requirements.txt format.", callback=callback)(f)
 
+def dev_only_flag(f):
+    def callback(ctx, param, value):
+        state = ctx.ensure_object(State)
+        if value:
+            state.lockoptions.dev_only = value
+        return value
+    return option("--dev-only", default=False, is_flag=True, expose_value=False,
+                  help="Emit development dependencies *only* (overrides --dev)", callback=callback)(f)
 
 def code_option(f):
     def callback(ctx, param, value):
@@ -397,6 +410,7 @@ def uninstall_options(f):
 def lock_options(f):
     f = install_base_options(f)
     f = requirements_flag(f)
+    f = dev_only_flag(f)
     return f
 
 

pipenv/core.py

@@ -772,9 +772,9 @@ def batch_install(deps_list, procs, failed_deps_queue,
 
 def do_install_dependencies(
     dev=False,
-    only=False,
+    dev_only=False,
     bare=False,
-    requirements=False,
+    emit_requirements=False,
     allow_global=False,
     ignore_hashes=False,
     skip_lock=False,
@@ -785,11 +785,11 @@ def do_install_dependencies(
     """"
     Executes the install functionality.
 
-    If requirements is True, simply spits out a requirements format to stdout.
+    If emit_requirements is True, simply spits out a requirements format to stdout.
     """
 
     from six.moves import queue
-    if requirements:
+    if emit_requirements:
         bare = True
     # Load the lockfile if it exists, or if only is being used (e.g. lock is being used).
     if skip_lock or only or not project.lockfile_exists:
@@ -812,14 +812,14 @@ def do_install_dependencies(
             )
     # Allow pip to resolve dependencies when in skip-lock mode.
     no_deps = not skip_lock  # skip_lock true, no_deps False, pip resolves deps
-    deps_list = list(lockfile.get_requirements(dev=dev, only=requirements))
-    if requirements:
+    dev = dev or dev_only
+    deps_list = list(lockfile.get_requirements(dev=dev, only=dev_only))
+    if emit_requirements:
         index_args = prepare_pip_source_args(project.sources)
         index_args = " ".join(index_args).replace(" -", "\n-")
         deps = [
             req.as_line(sources=False, include_hashes=False) for req in deps_list
         ]
-        # Output only default dependencies
         click.echo(index_args)
         click.echo(
             "\n".join(sorted(deps))
@@ -1171,7 +1171,8 @@ def do_purge(bare=False, downloads=False, allow_global=False):
 
 def do_init(
     dev=False,
-    requirements=False,
+    dev_only=False,
+    emit_requirements=False,
     allow_global=False,
     ignore_pipfile=False,
     skip_lock=False,
@@ -1276,7 +1277,8 @@ def do_init(
             )
     do_install_dependencies(
         dev=dev,
-        requirements=requirements,
+        dev_only=dev_only,
+        emit_requirements=emit_requirements,
         allow_global=allow_global,
         skip_lock=skip_lock,
         concurrent=concurrent,
@@ -1855,7 +1857,7 @@ def do_install(
     lock=True,
     ignore_pipfile=False,
     skip_lock=False,
-    requirements=False,
+    requirementstxt=False,
     sequential=False,
     pre=False,
     code=False,
@@ -1878,7 +1880,7 @@ def do_install(
     package_args = [p for p in packages if p] + [p for p in editable_packages if p]
     skip_requirements = False
     # Don't search for requirements.txt files if the user provides one
-    if requirements or package_args or project.pipfile_exists:
+    if requirementstxt or package_args or project.pipfile_exists:
         skip_requirements = True
     concurrent = not sequential
     # Ensure that virtualenv is available and pipfile are available
@@ -1903,7 +1905,7 @@ def do_install(
         pre = project.settings.get("allow_prereleases")
     if not keep_outdated:
         keep_outdated = project.settings.get("keep_outdated")
-    remote = requirements and is_valid_url(requirements)
+    remote = requirementstxt and is_valid_url(requirementstxt)
     # Warn and exit if --system is used without a pipfile.
     if (system and package_args) and not (PIPENV_VIRTUALENV):
         raise exceptions.SystemUsageError
@@ -1922,17 +1924,17 @@ def do_install(
             prefix="pipenv-", suffix="-requirement.txt", dir=requirements_directory
         )
         temp_reqs = fd.name
-        requirements_url = requirements
+        requirements_url = requirementstxt
         # Download requirements file
         try:
-            download_file(requirements, temp_reqs)
+            download_file(requirements_url, temp_reqs)
         except IOError:
             fd.close()
             os.unlink(temp_reqs)
             click.echo(
                 crayons.red(
                     u"Unable to find requirements file at {0}.".format(
-                        crayons.normal(requirements)
+                        crayons.normal(requirements_url)
                     )
                 ),
                 err=True,
@@ -1941,9 +1943,9 @@ def do_install(
         finally:
             fd.close()
         # Replace the url with the temporary requirements file
-        requirements = temp_reqs
+        requirementstxt = temp_reqs
         remote = True
-    if requirements:
+    if requirementstxt:
         error, traceback = None, None
         click.echo(
             crayons.normal(
@@ -1952,10 +1954,10 @@ def do_install(
             err=True,
         )
         try:
-            import_requirements(r=project.path_to(requirements), dev=dev)
+            import_requirements(r=project.path_to(requirementstxt), dev=dev)
         except (UnicodeDecodeError, PipError) as e:
             # Don't print the temp file path if remote since it will be deleted.
-            req_path = requirements_url if remote else project.path_to(requirements)
+            req_path = requirements_url if remote else project.path_to(requirementstxt)
             error = (
                 u"Unexpected syntax in {0}. Are you sure this is a "
                 "requirements.txt style file?".format(req_path)

pipenv/vendor/requirementslib/models/lockfile.py

@@ -263,6 +263,8 @@ def get_requirements(self, dev=True, only=False):
         """Produces a generator which generates requirements from the desired section.
 
         :param bool dev: Indicates whether to use dev requirements, defaults to False
+        :param bool only: When dev is True, indicates whether to use *only* dev
+                          requirements, defaults to False
         :return: Requirements from the relevant the relevant pipfile
         :rtype: :class:`~requirementslib.models.requirements.Requirement`
         """