diff --git a/plugins/flytekit-papermill/flytekitplugins/papermill/task.py b/plugins/flytekit-papermill/flytekitplugins/papermill/task.py index 00ec942fcd..9e9535dec2 100644 --- a/plugins/flytekit-papermill/flytekitplugins/papermill/task.py +++ b/plugins/flytekit-papermill/flytekitplugins/papermill/task.py @@ -86,6 +86,13 @@ class NotebookTask(PythonInstanceTask[T]): Users can access these notebooks after execution of the task locally or from remote servers. + .. note: + + By default, print statements in your notebook won't be transmitted to the pod logs/stdout. If you would + like to have logs forwarded as the notebook executes, pass the stream_logs argument. Note that notebook + logs can be quite verbose, so ensure you are prepared for any downstream log ingestion costs + (e.g., cloudwatch) + .. todo: Implicit extraction of SparkConfiguration from the notebook is not supported. @@ -139,10 +146,11 @@ def __init__( # Send the papermill logger to stdout so that it appears in pod logs. Note that papermill doesn't allow # injecting a logger, so we cannot redirect logs to the flyte child loggers (e.g., the userspace logger) - # and we instead send to stdout directly + # and inherit their settings, but we instead must send logs to stdout directly if self._stream_logs: papermill_logger = logging.getLogger("papermill") papermill_logger.addHandler(logging.StreamHandler(sys.stdout)) + # Papermill leaves the default level of DEBUG. We increase it here. papermill_logger.setLevel(logging.INFO) if not os.path.exists(self._notebook_path):