Merge remote-tracking branch 'upstream/master' into test-isort

dvc/command/base.py

@@ -40,12 +40,12 @@ def __init__(self, args):
 
     @property
     def default_targets(self):
-        """Default targets for `dvc repro` and `dvc pipeline`."""
-        from dvc.dvcfile import DVC_FILE
+        """Default targets for `dvc repro`."""
+        from dvc.dvcfile import PIPELINE_FILE
 
-        msg = "assuming default target '{}'.".format(DVC_FILE)
+        msg = "assuming default target '{}'.".format(PIPELINE_FILE)
         logger.warning(msg)
-        return [DVC_FILE]
+        return [PIPELINE_FILE]
 
     # Abstract methods that have to be implemented by any inheritance class
     def run(self):

dvc/command/pipeline.py

@@ -118,8 +118,10 @@ def _write_dot(self, target, commands, outs):
         logger.info(dot_file.getvalue())
 
     def run(self):
+        from dvc.dvcfile import DVC_FILE
+
         if not self.args.targets:
-            self.args.targets = self.default_targets
+            self.args.targets = [DVC_FILE]
 
         for target in self.args.targets:
             try:

dvc/remote/gdrive.py

@@ -44,17 +44,6 @@ def __init__(self, cred_location):
         super().__init__(message)
 
 
-def _extract(exc, field):
-    from pydrive2.files import ApiRequestError
-
-    assert isinstance(exc, ApiRequestError)
-
-    # https://cloud.google.com/storage/docs/json_api/v1/status-codes#errorformat
-    return (
-        exc.error["errors"][0].get(field, "") if "errors" in exc.error else ""
-    )
-
-
 def _gdrive_retry(func):
     def should_retry(exc):
         from pydrive2.files import ApiRequestError
@@ -68,7 +57,7 @@ def should_retry(exc):
             result = True
 
         if error_code == 403:
-            result = _extract(exc, "reason") in [
+            result = exc.GetField("reason") in [
                 "userRateLimitExceeded",
                 "rateLimitExceeded",
             ]
@@ -396,14 +385,15 @@ def _gdrive_download_file(
         param = {"id": item_id}
         # it does not create a file on the remote
         gdrive_file = self._drive.CreateFile(param)
-        bar_format = (
-            "Downloading {desc:{ncols_desc}.{ncols_desc}}... "
-            + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024)
-        )
+
         with Tqdm(
-            bar_format=bar_format, desc=progress_desc, disable=no_progress_bar
-        ):
-            gdrive_file.GetContentFile(to_file)
+            desc=progress_desc,
+            disable=no_progress_bar,
+            bytes=True,
+            # explicit `bar_format` as `total` will be set by `update_to`
+            bar_format=Tqdm.BAR_FMT_DEFAULT,
+        ) as pbar:
+            gdrive_file.GetContentFile(to_file, callback=pbar.update_to)
 
     @_gdrive_retry
     def _gdrive_delete_file(self, item_id):
@@ -420,7 +410,7 @@ def _gdrive_delete_file(self, item_id):
             if (
                 http_error_code == 403
                 and self._list_params["corpora"] == "drive"
-                and _extract(exc, "location") == "file.permissions"
+                and exc.GetField("location") == "file.permissions"
             ):
                 raise DvcException(
                     "Insufficient permissions to {}. You should have {} "

dvc/repo/plot/template.py

@@ -23,6 +23,13 @@ def __init__(self, template_path):
         )
 
 
+class NoFieldInDataError(DvcException):
+    def __init__(self, field_name):
+        super().__init__(
+            "Field '{}' does not exist in provided data.".format(field_name)
+        )
+
+
 class Template:
     INDENT = 4
     SEPARATORS = (",", ": ")
@@ -90,6 +97,11 @@ def fill(
         with open(template_path, "r") as fobj:
             result_content = fobj.read()
 
+        if x_field:
+            Template._check_field_exists(data, x_field)
+        if y_field:
+            Template._check_field_exists(data, y_field)
+
         result_content = Template._replace_data_anchors(
             result_content, data, priority_datafile
         )
@@ -100,6 +112,14 @@ def fill(
 
         return result_content
 
+    @staticmethod
+    def _check_field_exists(data, field):
+        for file, data_points in data.items():
+            if not any(
+                field in data_point.keys() for data_point in data_points
+            ):
+                raise NoFieldInDataError(field)
+
     @staticmethod
     def _replace_metadata_anchors(
         result_content, title, x_field, x_title, y_field, y_title

setup.py

@@ -87,7 +87,7 @@ def run(self):
 # Extra dependencies for remote integrations
 
 gs = ["google-cloud-storage==1.19.0"]
-gdrive = ["pydrive2>=1.4.10"]
+gdrive = ["pydrive2>=1.4.11"]
 s3 = ["boto3>=1.9.201"]
 azure = ["azure-storage-blob==2.1.0"]
 oss = ["oss2==2.6.1"]

tests/func/test_plot.py

@@ -18,6 +18,7 @@
 )
 from dvc.repo.plot.template import (
     NoDataForTemplateError,
+    NoFieldInDataError,
     TemplateNotFoundError,
 )
 
@@ -502,3 +503,15 @@ def test_plot_yaml(tmp_dir, scm, dvc):
         {"val": 2, PlotData.INDEX_FIELD: 0, "rev": "workspace"},
         {"val": 3, PlotData.INDEX_FIELD: 1, "rev": "workspace"},
     ]
+
+
+def test_raise_on_wrong_field(tmp_dir, scm, dvc):
+    metric = [{"val": 2}, {"val": 3}]
+    _write_json(tmp_dir, metric, "metric.json")
+    _run_with_metric(tmp_dir, "metric.json", "first run")
+
+    with pytest.raises(NoFieldInDataError):
+        dvc.plot("metric.json", x_field="no_val")
+
+    with pytest.raises(NoFieldInDataError):
+        dvc.plot("metric.json", y_field="no_val")

tests/func/test_repro.py

@@ -18,7 +18,7 @@
 from mock import patch
 
 from dvc.compat import fspath
-from dvc.dvcfile import Dvcfile
+from dvc.dvcfile import DVC_FILE, Dvcfile
 from dvc.exceptions import (
     CyclicGraphError,
     ReproductionError,
@@ -406,11 +406,11 @@ def test(self):
             self.assertEqual(ret, 0)
 
         ret = main(
-            ["run", "--no-exec", "--single-stage", "-f", "Dvcfile"] + deps
+            ["run", "--no-exec", "--single-stage", "-f", DVC_FILE] + deps
         )
         self.assertEqual(ret, 0)
 
-        ret = main(["repro", "--dry"])
+        ret = main(["repro", "--dry", DVC_FILE])
         self.assertEqual(ret, 0)
 
 
@@ -819,7 +819,7 @@ def test(self):
 
         os.unlink(bar)
 
-        ret = main(["repro", "-c", dname])
+        ret = main(["repro", "-c", dname, DVC_FILE])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(foo))
         self.assertTrue(os.path.isfile(bar))
@@ -1457,7 +1457,7 @@ def repro_dir(tmp_dir, dvc, run_copy):
     assert dir_file_copy.read_text() == "dir file content"
     stages["dir_file_copy"] = stage
 
-    last_stage = tmp_dir / "dir" / "Dvcfile"
+    last_stage = tmp_dir / "dir" / DVC_FILE
     stage = dvc.run(
         fname=fspath(last_stage),
         deps=[fspath(origin_copy_2), fspath(dir_file_copy)],

tests/unit/command/test_repro.py

@@ -1,5 +1,6 @@
 from dvc.cli import parse_args
 from dvc.command.repro import CmdRepro
+from dvc.dvcfile import PIPELINE_FILE
 
 default_arguments = {
     "all_pipelines": False,
@@ -19,7 +20,7 @@ def test_default_arguments(dvc, mocker):
     cmd = CmdRepro(parse_args(["repro"]))
     mocker.patch.object(cmd.repo, "reproduce")
     cmd.run()
-    cmd.repo.reproduce.assert_called_with("Dvcfile", **default_arguments)
+    cmd.repo.reproduce.assert_called_with(PIPELINE_FILE, **default_arguments)
 
 
 def test_downstream(dvc, mocker):
@@ -28,4 +29,4 @@ def test_downstream(dvc, mocker):
     cmd.run()
     arguments = default_arguments.copy()
     arguments.update({"downstream": True})
-    cmd.repo.reproduce.assert_called_with("Dvcfile", **arguments)
+    cmd.repo.reproduce.assert_called_with(PIPELINE_FILE, **arguments)