From 188228f36f69eae43dbc28cb5e61482f26ac0960 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 23 Jan 2024 16:33:49 +0100 Subject: [PATCH 1/2] [3.12] gh-114440: Close writer pipe in multiprocessing.Queue, not concurrent.futures This was left out of the 3.12 backport for three related issues: - gh-107219 (which adds `self.call_queue._writer.close()` to `_ExecutorManagerThread` in `concurrent.futures`) - gh-109370 (which changes this to be only called on Windows) - gh-109047 (which moves the call to `multiprocessing.Queue`'s `_terminate_broken`) Without this change, ProcessPoolExecutor sometimes hangs on Windows when a worker process is terminated. Co-authored-by: Victor Stinner Co-authored-by: Serhiy Storchaka --- Lib/concurrent/futures/process.py | 5 ----- Lib/multiprocessing/queues.py | 5 +++++ .../Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst | 6 ++++++ 3 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 33e62fe231e187..0e452883963c17 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -524,11 +524,6 @@ def _terminate_broken(self, cause): self.call_queue._terminate_broken() - # gh-107219: Close the connection writer which can unblock - # Queue._feed() if it was stuck in send_bytes(). - if sys.platform == 'win32': - self.call_queue._writer.close() - # clean up resources self._join_executor_internals(broken=True) diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index d36de75749f4ed..852ae87b276861 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -164,6 +164,11 @@ def _terminate_broken(self): # gh-94777: Prevent queue writing to a pipe which is no longer read. self._reader.close() + # gh-107219: Close the connection writer which can unblock + # Queue._feed() if it was stuck in send_bytes(). + if sys.platform == 'win32': + self._writer.close() + self.close() self.join_thread() diff --git a/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst b/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst new file mode 100644 index 00000000000000..dba3ad012d84ad --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst @@ -0,0 +1,6 @@ +On Windows, closing the connection writer when cleaning up a broken +:cls:`multiprocessing.Queue` queue is now done for all queues, rather than +only in :mod:`concurrent.futures` manager thread. +This can prevent a deadlock when a ``multiprocessing`` worker process terminates +without cleaning up. +This completes the backport of patches by Victor Stinner and Serhiy Storchaka. From 62af0b7dd037bb9c76e41faaa7cfa3647094580d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 23 Jan 2024 18:27:12 +0200 Subject: [PATCH 2/2] Update Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst --- .../next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst b/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst index dba3ad012d84ad..7231a3c4ef8325 100644 --- a/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst +++ b/Misc/NEWS.d/next/Library/2023-09-25-02-11-14.gh-issue-114440.b2TrqG.rst @@ -1,5 +1,5 @@ On Windows, closing the connection writer when cleaning up a broken -:cls:`multiprocessing.Queue` queue is now done for all queues, rather than +:class:`multiprocessing.Queue` queue is now done for all queues, rather than only in :mod:`concurrent.futures` manager thread. This can prevent a deadlock when a ``multiprocessing`` worker process terminates without cleaning up.