feat(ingest): add ingest --no-progress option

docs/cli.md

@@ -93,6 +93,7 @@ Usage: datahub [datahub-options] ingest [command-options]
 
 Command Options:
   -c / --config             Config file in .toml or .yaml format
+  -q / --quiet              Mute intermediate ingestion reports
   -n / --dry-run            Perform a dry run of the ingestion, essentially skipping writing to sink
   --preview                 Perform limited ingestion from the source to the sink to get a quick preview
   --preview-workunits       The number of workunits to produce for preview

metadata-ingestion/src/datahub/cli/ingest_cli.py

@@ -48,6 +48,14 @@ def ingest() -> None:
     help="Config file in .toml or .yaml format.",
     required=True,
 )
+@click.option(
+    "-q",
+    "--quiet",
+    type=bool,
+    is_flag=True,
+    default=False,
+    help="Mute intermediate ingestion reports.",
+)
 @click.option(
     "-n",
     "--dry-run",
@@ -99,6 +107,7 @@ def ingest() -> None:
 )
 @telemetry.with_telemetry(
     capture_kwargs=[
+        "quiet",
         "dry_run",
         "preview",
         "strict_warnings",
@@ -109,6 +118,7 @@ def ingest() -> None:
 )
 def run(
     config: str,
+    quiet: bool,
     dry_run: bool,
     preview: bool,
     strict_warnings: bool,
@@ -162,6 +172,7 @@ async def run_ingestion_and_check_upgrade() -> int:
         # logger.debug(f"Using config: {pipeline_config}")
         pipeline = Pipeline.create(
             pipeline_config,
+            quiet,
             dry_run,
             preview,
             preview_workunits,

metadata-ingestion/src/datahub/ingestion/run/pipeline.py

@@ -168,13 +168,15 @@ class Pipeline:
     def __init__(
         self,
         config: PipelineConfig,
+        quiet: bool = False,
         dry_run: bool = False,
         preview_mode: bool = False,
         preview_workunits: int = 10,
         report_to: Optional[str] = None,
         no_default_report: bool = False,
     ):
         self.config = config
+        self.quiet = quiet
         self.dry_run = dry_run
         self.preview_mode = preview_mode
         self.preview_workunits = preview_workunits
@@ -379,7 +381,7 @@ def run(self) -> None:
                     self.preview_workunits if self.preview_mode else None,
                 ):
                     try:
-                        if self._time_to_print():
+                        if self._time_to_print() and not self.quiet:
                             self.pretty_print_summary(currently_running=True)
                     except Exception as e:
                         logger.warning(f"Failed to print summary {e}")