Mailman 3 July 2026 - Python-checkins

[3.15] gh-47005: fix do_open() to let regular headers override unredirected … (GH-146506) (#153377)
by orsenthil July 9, 2026

July 9, 2026

https://github.com/python/cpython/commit/9fb25cbd41410fb9bdf230d6242f035888… commit: 9fb25cbd41410fb9bdf230d6242f0358885405ae branch: 3.15 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: orsenthil <senthil(a)python.org> date: 2026-07-09T04:15:30Z summary: [3.15] gh-47005: fix do_open() to let regular headers override unredirected … (GH-146506) (#153377) gh-47005: fix do_open() to let regular headers override unredirected … (GH-146506) AbstractHTTPHandler.do_open() was building the request header dict by starting with unredirected_hdrs and only inserting regular headers that were not already present, giving unredirected headers priority. This contradicts get_header() and header_items(), both of which give regular headers the higher priority. Fix by unconditionally updating with req.headers so that a header set via add_header() always overrides one set via add_unredirected_header(). --------- (cherry picked from commit 6d386684ad69f42cb57c72fe0e0ffcec82ec7e12) Co-authored-by: CHINMAY <89741289+Das-Chinmay(a)users.noreply.github.com> Co-authored-by: Senthil Kumaran <senthil(a)python.org> files: A Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst M Lib/test/test_urllib2.py M Lib/urllib/request.py diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 3a77b9e5ab79289..d2fd111f6d9de02 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -980,6 +980,35 @@ def test_http(self): self.assertEqual(req.unredirected_hdrs["Host"], "baz") self.assertEqual(req.unredirected_hdrs["Spam"], "foo") + def test_http_header_priority(self): + # gh-47005: regular headers set via add_header() must override + # unredirected headers with the same name in do_open(), consistent + # with get_header() and header_items(). + cases = [ + ("Content-Type", "application/json", "application/x-www-form-urlencoded"), + ("Content-Length", "99", "0"), + ("Host", "override.example.com", "internal.example.com"), + ("Authorization", "Bearer user-token", "Basic stale="), + ("Cookie", "a=1", "b=2"), + ("User-Agent", "MyApp/1.0", "Python-urllib/test"), + ] + h = urllib.request.AbstractHTTPHandler() + h.parent = MockOpener() + + for key, regular, unredirected in cases: + req = Request("http://example.com/", headers={key: regular}) + req.timeout = None + req.add_unredirected_header(key, unredirected) + + http = MockHTTPClass() + h.do_open(http, req) + + sent_headers = dict(http.req_headers) + self.assertEqual(sent_headers[key], regular) + # key is capitalized by add_header() and add_unredirected_header() calls + self.assertEqual(req.get_header(key.capitalize()), regular) + self.assertEqual(dict(req.header_items())[key.capitalize()], regular) + def test_http_body_file(self): # A regular file - chunked encoding is used unless Content Length is # already set. diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index f5f17f223a45853..660301fef612588 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1293,8 +1293,7 @@ def do_open(self, http_class, req, **http_conn_args): h.set_debuglevel(self._debuglevel) headers = dict(req.unredirected_hdrs) - headers.update({k: v for k, v in req.headers.items() - if k not in headers}) + headers.update(req.headers) # TODO(jhylton): Should this be redesigned to handle # persistent connections? diff --git a/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst b/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst new file mode 100644 index 000000000000000..646367f0fa069d3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst @@ -0,0 +1,4 @@ +Fix :meth:`!urllib.request.AbstractHTTPHandler.do_open` to give regular +headers set via :meth:`~urllib.request.Request.add_header` priority over +unredirected headers, consistent with :meth:`~urllib.request.Request.get_header` +and :meth:`~urllib.request.Request.header_items`.

1 0

[3.13] gh-47005: fix do_open() to let regular headers override unredirected … (GH-146506) (#153379)
by orsenthil July 9, 2026

July 9, 2026

https://github.com/python/cpython/commit/ff3fca10ba3dfe4385e5b1ecc57082fef7… commit: ff3fca10ba3dfe4385e5b1ecc57082fef79c9db0 branch: 3.13 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: orsenthil <senthil(a)python.org> date: 2026-07-09T04:06:48Z summary: [3.13] gh-47005: fix do_open() to let regular headers override unredirected … (GH-146506) (#153379) gh-47005: fix do_open() to let regular headers override unredirected … (GH-146506) AbstractHTTPHandler.do_open() was building the request header dict by starting with unredirected_hdrs and only inserting regular headers that were not already present, giving unredirected headers priority. This contradicts get_header() and header_items(), both of which give regular headers the higher priority. Fix by unconditionally updating with req.headers so that a header set via add_header() always overrides one set via add_unredirected_header(). --------- (cherry picked from commit 6d386684ad69f42cb57c72fe0e0ffcec82ec7e12) Co-authored-by: CHINMAY <89741289+Das-Chinmay(a)users.noreply.github.com> Co-authored-by: Senthil Kumaran <senthil(a)python.org> files: A Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst M Lib/test/test_urllib2.py M Lib/urllib/request.py diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 725fe3493c58ae7..d94021b31b19ff3 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -950,6 +950,35 @@ def test_http(self): self.assertEqual(req.unredirected_hdrs["Host"], "baz") self.assertEqual(req.unredirected_hdrs["Spam"], "foo") + def test_http_header_priority(self): + # gh-47005: regular headers set via add_header() must override + # unredirected headers with the same name in do_open(), consistent + # with get_header() and header_items(). + cases = [ + ("Content-Type", "application/json", "application/x-www-form-urlencoded"), + ("Content-Length", "99", "0"), + ("Host", "override.example.com", "internal.example.com"), + ("Authorization", "Bearer user-token", "Basic stale="), + ("Cookie", "a=1", "b=2"), + ("User-Agent", "MyApp/1.0", "Python-urllib/test"), + ] + h = urllib.request.AbstractHTTPHandler() + h.parent = MockOpener() + + for key, regular, unredirected in cases: + req = Request("http://example.com/", headers={key: regular}) + req.timeout = None + req.add_unredirected_header(key, unredirected) + + http = MockHTTPClass() + h.do_open(http, req) + + sent_headers = dict(http.req_headers) + self.assertEqual(sent_headers[key], regular) + # key is capitalized by add_header() and add_unredirected_header() calls + self.assertEqual(req.get_header(key.capitalize()), regular) + self.assertEqual(dict(req.header_items())[key.capitalize()], regular) + def test_http_body_file(self): # A regular file - chunked encoding is used unless Content Length is # already set. diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 3d864f1d92fe7c0..42147f31d968c77 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1289,8 +1289,7 @@ def do_open(self, http_class, req, **http_conn_args): h.set_debuglevel(self._debuglevel) headers = dict(req.unredirected_hdrs) - headers.update({k: v for k, v in req.headers.items() - if k not in headers}) + headers.update(req.headers) # TODO(jhylton): Should this be redesigned to handle # persistent connections? diff --git a/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst b/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst new file mode 100644 index 000000000000000..646367f0fa069d3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst @@ -0,0 +1,4 @@ +Fix :meth:`!urllib.request.AbstractHTTPHandler.do_open` to give regular +headers set via :meth:`~urllib.request.Request.add_header` priority over +unredirected headers, consistent with :meth:`~urllib.request.Request.get_header` +and :meth:`~urllib.request.Request.header_items`.

1 0

gh-47005: fix do_open() to let regular headers override unredirected … (#146506)
by orsenthil July 9, 2026

July 9, 2026

https://github.com/python/cpython/commit/6d386684ad69f42cb57c72fe0e0ffcec82… commit: 6d386684ad69f42cb57c72fe0e0ffcec82ec7e12 branch: main author: CHINMAY <89741289+Das-Chinmay(a)users.noreply.github.com> committer: orsenthil <senthil(a)python.org> date: 2026-07-08T20:40:18-07:00 summary: gh-47005: fix do_open() to let regular headers override unredirected … (#146506) * gh-47005: fix do_open() to let regular headers override unredirected headers AbstractHTTPHandler.do_open() was building the request header dict by starting with unredirected_hdrs and only inserting regular headers that were not already present, giving unredirected headers priority. This contradicts get_header() and header_items(), both of which give regular headers the higher priority. Fix by unconditionally updating with req.headers so that a header set via add_header() always overrides one set via add_unredirected_header(). --------- Co-authored-by: Senthil Kumaran <senthil(a)python.org> files: A Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst M Lib/test/test_urllib2.py M Lib/urllib/request.py diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 3a77b9e5ab79289..d2fd111f6d9de02 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -980,6 +980,35 @@ def test_http(self): self.assertEqual(req.unredirected_hdrs["Host"], "baz") self.assertEqual(req.unredirected_hdrs["Spam"], "foo") + def test_http_header_priority(self): + # gh-47005: regular headers set via add_header() must override + # unredirected headers with the same name in do_open(), consistent + # with get_header() and header_items(). + cases = [ + ("Content-Type", "application/json", "application/x-www-form-urlencoded"), + ("Content-Length", "99", "0"), + ("Host", "override.example.com", "internal.example.com"), + ("Authorization", "Bearer user-token", "Basic stale="), + ("Cookie", "a=1", "b=2"), + ("User-Agent", "MyApp/1.0", "Python-urllib/test"), + ] + h = urllib.request.AbstractHTTPHandler() + h.parent = MockOpener() + + for key, regular, unredirected in cases: + req = Request("http://example.com/", headers={key: regular}) + req.timeout = None + req.add_unredirected_header(key, unredirected) + + http = MockHTTPClass() + h.do_open(http, req) + + sent_headers = dict(http.req_headers) + self.assertEqual(sent_headers[key], regular) + # key is capitalized by add_header() and add_unredirected_header() calls + self.assertEqual(req.get_header(key.capitalize()), regular) + self.assertEqual(dict(req.header_items())[key.capitalize()], regular) + def test_http_body_file(self): # A regular file - chunked encoding is used unless Content Length is # already set. diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index f5f17f223a45853..660301fef612588 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1293,8 +1293,7 @@ def do_open(self, http_class, req, **http_conn_args): h.set_debuglevel(self._debuglevel) headers = dict(req.unredirected_hdrs) - headers.update({k: v for k, v in req.headers.items() - if k not in headers}) + headers.update(req.headers) # TODO(jhylton): Should this be redesigned to handle # persistent connections? diff --git a/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst b/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst new file mode 100644 index 000000000000000..646367f0fa069d3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst @@ -0,0 +1,4 @@ +Fix :meth:`!urllib.request.AbstractHTTPHandler.do_open` to give regular +headers set via :meth:`~urllib.request.Request.add_header` priority over +unredirected headers, consistent with :meth:`~urllib.request.Request.get_header` +and :meth:`~urllib.request.Request.header_items`.

1 0

[3.15] gh-151213: Document asyncio debugging tools (GH-151392) (#153370)
by pablogsal July 8, 2026

July 8, 2026

https://github.com/python/cpython/commit/c512e8f0fb6b78da7fe91aa267700ea516… commit: c512e8f0fb6b78da7fe91aa267700ea516dfd1f6 branch: 3.15 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2026-07-08T23:25:18Z summary: [3.15] gh-151213: Document asyncio debugging tools (GH-151392) (#153370) gh-151213: Document asyncio debugging tools (GH-151392) (cherry picked from commit b2d8db1ac818e74ffe666146335e9e6c7f1f3f56) Co-authored-by: Bartosz Sławecki <bartosz(a)ilikepython.com> files: A Doc/library/asyncio-tools.rst M Doc/library/asyncio-graph.rst M Doc/library/asyncio.rst diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 5f642a32bf75c2..1a8cdcf6a9e8cb 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -4,7 +4,7 @@ .. _asyncio-graph: ======================== -Call Graph Introspection +Call graph introspection ======================== **Source code:** :source:`Lib/asyncio/graph.py` @@ -17,6 +17,12 @@ a suspended *future*. These utilities and the underlying machinery can be used from within a Python program or by external profilers and debuggers. +.. seealso:: + + :ref:`asyncio-introspection-tools` + Command-line tools for inspecting tasks in another running Python + process. + .. versionadded:: 3.14 diff --git a/Doc/library/asyncio-tools.rst b/Doc/library/asyncio-tools.rst new file mode 100644 index 00000000000000..1782640e83f53a --- /dev/null +++ b/Doc/library/asyncio-tools.rst @@ -0,0 +1,157 @@ +.. currentmodule:: asyncio + +.. _asyncio-introspection-tools: + +================================ +Command-line introspection tools +================================ + +**Source code:** :source:`Lib/asyncio/tools.py` + +------------------------------------- + +The :mod:`!asyncio` module can be invoked as a script via ``python -m +asyncio`` to inspect the task graph of another running Python process without +modifying it or restarting it. The :mod:`!asyncio.tools` submodule implements +this interface. + +The following commands inspect the process identified by ``PID``: + +.. code-block:: shell-session + + $ python -m asyncio pstree [--retries N] PID + $ python -m asyncio ps [--retries N] PID + +The commands read the target process state without executing any code in it. +They are only available on supported platforms and may require permission to +inspect another process. See the :ref:`permission-requirements <permission-requirements>` for details. + +.. seealso:: + + :ref:`asyncio-graph` + Programmatic APIs for inspecting the async call graph of a task or + future in the current process. + +The command examples below use this program, which creates a task hierarchy +suitable for inspection and prints its process ID: + +.. code-block:: python + :caption: example.py + + import asyncio + import os + + async def play(track): + await asyncio.sleep(3600) + print(f"🎵 Finished: {track}") + + async def album(name, tracks): + async with asyncio.TaskGroup() as tg: + for track in tracks: + tg.create_task(play(track), name=track) + + async def main(): + print(f"PID: {os.getpid()}") + async with asyncio.TaskGroup() as tg: + tg.create_task( + album("Sundowning", ["TNDNBTG", "Levitate"]), + name="Sundowning", + ) + tg.create_task( + album("TMBTE", ["DYWTYLM", "Aqua Regia"]), + name="TMBTE", + ) + + asyncio.run(main()) + +Run the program in one terminal and leave it running: + +.. code-block:: shell-session + + $ python example.py + PID: 12345 + +Then pass the printed process ID to the commands from another terminal. +Thread IDs, task IDs, file paths, and line numbers vary between runs and +source layouts. + +.. versionadded:: 3.14 + +Command-line options +==================== + +.. option:: pstree PID + + Display task and coroutine relationships as a tree. Each task is shown + with its full coroutine stack, nested under the task (if any) that is + awaiting it. This subcommand is useful for quickly identifying which branch + of a task hierarchy is blocked and where in its coroutine stack execution + has paused: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + └── (T) Task-1 + └── main example.py:12 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) Sundowning + │ └── album example.py:7 + │ └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + │ └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + │ ├── (T) TNDNBTG + │ │ └── play example.py:4 + │ │ └── sleep Lib/asyncio/tasks.py:702 + │ └── (T) Levitate + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) TMBTE + └── album example.py:7 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) DYWTYLM + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) Aqua Regia + └── play example.py:4 + └── sleep Lib/asyncio/tasks.py:702 + + If the await graph contains a cycle, ``pstree`` reports an error instead + of printing a tree. A cycle in the await graph is unusual and typically + indicates a programming error: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + ERROR: await-graph contains cycles - cannot print a tree! + + cycle: Task-2 → Task-3 → Task-2 + +.. option:: ps PID + + Display a flat table of all pending tasks in the process *PID*. Each row + shows the event-loop thread ID, task ID and name, coroutine stack, and the + awaiting task's stack, name, and ID, if any. + + This subcommand prints all tasks regardless of whether the await graph + contains cycles: + + .. code-block:: shell-session + + $ python -m asyncio ps 12345 + tid task id task name coroutine stack awaiter chain awaiter name awaiter id + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 18445801 0x10a456060 Task-1 TaskGroup._aexit -> TaskGroup.__aexit__ -> main 0x0 + 18445801 0x10a439f60 Sundowning TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a439d70 TMBTE TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a2a3a80 TNDNBTG sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2a38a0 Levitate sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2d7150 DYWTYLM sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 + 18445801 0x10a6bdaa0 Aqua Regia sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 + +.. option:: --retries N + + Retry failed attempts to inspect the target process up to *N* times. This + can help when the target process changes while its state is being read. + + .. versionadded:: 3.15 diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 90a465f3e1d3af..8d19f49f6a0621 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -47,6 +47,13 @@ asyncio provides a set of **high-level** APIs to: * :ref:`synchronize <asyncio-sync>` concurrent code; +For **introspection**, asyncio provides APIs and tools for: + +* inspecting the :ref:`async call graph <asyncio-graph>` of tasks and futures; + +* inspecting tasks in another running Python process with + :ref:`command-line tools <asyncio-introspection-tools>`; + Additionally, there are **low-level** APIs for *library and framework developers* to: @@ -108,7 +115,13 @@ for full functionality and the latest features. asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst + +.. toctree:: + :caption: Introspection APIs + :maxdepth: 1 + asyncio-graph.rst + asyncio-tools.rst .. toctree:: :caption: Low-level APIs

1 0

[3.14] gh-151213: Document asyncio debugging tools (GH-151392) (#153371)
by pablogsal July 8, 2026

July 8, 2026

https://github.com/python/cpython/commit/0f529b1ca2b907ba6917dfdd298b3ff6cd… commit: 0f529b1ca2b907ba6917dfdd298b3ff6cd8bd21a branch: 3.14 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2026-07-08T23:23:35Z summary: [3.14] gh-151213: Document asyncio debugging tools (GH-151392) (#153371) gh-151213: Document asyncio debugging tools (GH-151392) (cherry picked from commit b2d8db1ac818e74ffe666146335e9e6c7f1f3f56) Co-authored-by: Bartosz Sławecki <bartosz(a)ilikepython.com> files: A Doc/library/asyncio-tools.rst M Doc/library/asyncio-graph.rst M Doc/library/asyncio.rst diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 5f642a32bf75c2..1a8cdcf6a9e8cb 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -4,7 +4,7 @@ .. _asyncio-graph: ======================== -Call Graph Introspection +Call graph introspection ======================== **Source code:** :source:`Lib/asyncio/graph.py` @@ -17,6 +17,12 @@ a suspended *future*. These utilities and the underlying machinery can be used from within a Python program or by external profilers and debuggers. +.. seealso:: + + :ref:`asyncio-introspection-tools` + Command-line tools for inspecting tasks in another running Python + process. + .. versionadded:: 3.14 diff --git a/Doc/library/asyncio-tools.rst b/Doc/library/asyncio-tools.rst new file mode 100644 index 00000000000000..1782640e83f53a --- /dev/null +++ b/Doc/library/asyncio-tools.rst @@ -0,0 +1,157 @@ +.. currentmodule:: asyncio + +.. _asyncio-introspection-tools: + +================================ +Command-line introspection tools +================================ + +**Source code:** :source:`Lib/asyncio/tools.py` + +------------------------------------- + +The :mod:`!asyncio` module can be invoked as a script via ``python -m +asyncio`` to inspect the task graph of another running Python process without +modifying it or restarting it. The :mod:`!asyncio.tools` submodule implements +this interface. + +The following commands inspect the process identified by ``PID``: + +.. code-block:: shell-session + + $ python -m asyncio pstree [--retries N] PID + $ python -m asyncio ps [--retries N] PID + +The commands read the target process state without executing any code in it. +They are only available on supported platforms and may require permission to +inspect another process. See the :ref:`permission-requirements <permission-requirements>` for details. + +.. seealso:: + + :ref:`asyncio-graph` + Programmatic APIs for inspecting the async call graph of a task or + future in the current process. + +The command examples below use this program, which creates a task hierarchy +suitable for inspection and prints its process ID: + +.. code-block:: python + :caption: example.py + + import asyncio + import os + + async def play(track): + await asyncio.sleep(3600) + print(f"🎵 Finished: {track}") + + async def album(name, tracks): + async with asyncio.TaskGroup() as tg: + for track in tracks: + tg.create_task(play(track), name=track) + + async def main(): + print(f"PID: {os.getpid()}") + async with asyncio.TaskGroup() as tg: + tg.create_task( + album("Sundowning", ["TNDNBTG", "Levitate"]), + name="Sundowning", + ) + tg.create_task( + album("TMBTE", ["DYWTYLM", "Aqua Regia"]), + name="TMBTE", + ) + + asyncio.run(main()) + +Run the program in one terminal and leave it running: + +.. code-block:: shell-session + + $ python example.py + PID: 12345 + +Then pass the printed process ID to the commands from another terminal. +Thread IDs, task IDs, file paths, and line numbers vary between runs and +source layouts. + +.. versionadded:: 3.14 + +Command-line options +==================== + +.. option:: pstree PID + + Display task and coroutine relationships as a tree. Each task is shown + with its full coroutine stack, nested under the task (if any) that is + awaiting it. This subcommand is useful for quickly identifying which branch + of a task hierarchy is blocked and where in its coroutine stack execution + has paused: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + └── (T) Task-1 + └── main example.py:12 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) Sundowning + │ └── album example.py:7 + │ └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + │ └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + │ ├── (T) TNDNBTG + │ │ └── play example.py:4 + │ │ └── sleep Lib/asyncio/tasks.py:702 + │ └── (T) Levitate + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) TMBTE + └── album example.py:7 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) DYWTYLM + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) Aqua Regia + └── play example.py:4 + └── sleep Lib/asyncio/tasks.py:702 + + If the await graph contains a cycle, ``pstree`` reports an error instead + of printing a tree. A cycle in the await graph is unusual and typically + indicates a programming error: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + ERROR: await-graph contains cycles - cannot print a tree! + + cycle: Task-2 → Task-3 → Task-2 + +.. option:: ps PID + + Display a flat table of all pending tasks in the process *PID*. Each row + shows the event-loop thread ID, task ID and name, coroutine stack, and the + awaiting task's stack, name, and ID, if any. + + This subcommand prints all tasks regardless of whether the await graph + contains cycles: + + .. code-block:: shell-session + + $ python -m asyncio ps 12345 + tid task id task name coroutine stack awaiter chain awaiter name awaiter id + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 18445801 0x10a456060 Task-1 TaskGroup._aexit -> TaskGroup.__aexit__ -> main 0x0 + 18445801 0x10a439f60 Sundowning TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a439d70 TMBTE TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a2a3a80 TNDNBTG sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2a38a0 Levitate sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2d7150 DYWTYLM sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 + 18445801 0x10a6bdaa0 Aqua Regia sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 + +.. option:: --retries N + + Retry failed attempts to inspect the target process up to *N* times. This + can help when the target process changes while its state is being read. + + .. versionadded:: 3.15 diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 90a465f3e1d3af..8d19f49f6a0621 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -47,6 +47,13 @@ asyncio provides a set of **high-level** APIs to: * :ref:`synchronize <asyncio-sync>` concurrent code; +For **introspection**, asyncio provides APIs and tools for: + +* inspecting the :ref:`async call graph <asyncio-graph>` of tasks and futures; + +* inspecting tasks in another running Python process with + :ref:`command-line tools <asyncio-introspection-tools>`; + Additionally, there are **low-level** APIs for *library and framework developers* to: @@ -108,7 +115,13 @@ for full functionality and the latest features. asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst + +.. toctree:: + :caption: Introspection APIs + :maxdepth: 1 + asyncio-graph.rst + asyncio-tools.rst .. toctree:: :caption: Low-level APIs

1 0

gh-151213: Document asyncio debugging tools (#151392)
by pablogsal July 8, 2026

July 8, 2026

https://github.com/python/cpython/commit/b2d8db1ac818e74ffe666146335e9e6c7f… commit: b2d8db1ac818e74ffe666146335e9e6c7f1f3f56 branch: main author: Bartosz Sławecki <bartosz(a)ilikepython.com> committer: pablogsal <Pablogsal(a)gmail.com> date: 2026-07-09T00:17:05+01:00 summary: gh-151213: Document asyncio debugging tools (#151392) files: A Doc/library/asyncio-tools.rst M Doc/library/asyncio-graph.rst M Doc/library/asyncio.rst diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 5f642a32bf75c2..1a8cdcf6a9e8cb 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -4,7 +4,7 @@ .. _asyncio-graph: ======================== -Call Graph Introspection +Call graph introspection ======================== **Source code:** :source:`Lib/asyncio/graph.py` @@ -17,6 +17,12 @@ a suspended *future*. These utilities and the underlying machinery can be used from within a Python program or by external profilers and debuggers. +.. seealso:: + + :ref:`asyncio-introspection-tools` + Command-line tools for inspecting tasks in another running Python + process. + .. versionadded:: 3.14 diff --git a/Doc/library/asyncio-tools.rst b/Doc/library/asyncio-tools.rst new file mode 100644 index 00000000000000..1782640e83f53a --- /dev/null +++ b/Doc/library/asyncio-tools.rst @@ -0,0 +1,157 @@ +.. currentmodule:: asyncio + +.. _asyncio-introspection-tools: + +================================ +Command-line introspection tools +================================ + +**Source code:** :source:`Lib/asyncio/tools.py` + +------------------------------------- + +The :mod:`!asyncio` module can be invoked as a script via ``python -m +asyncio`` to inspect the task graph of another running Python process without +modifying it or restarting it. The :mod:`!asyncio.tools` submodule implements +this interface. + +The following commands inspect the process identified by ``PID``: + +.. code-block:: shell-session + + $ python -m asyncio pstree [--retries N] PID + $ python -m asyncio ps [--retries N] PID + +The commands read the target process state without executing any code in it. +They are only available on supported platforms and may require permission to +inspect another process. See the :ref:`permission-requirements <permission-requirements>` for details. + +.. seealso:: + + :ref:`asyncio-graph` + Programmatic APIs for inspecting the async call graph of a task or + future in the current process. + +The command examples below use this program, which creates a task hierarchy +suitable for inspection and prints its process ID: + +.. code-block:: python + :caption: example.py + + import asyncio + import os + + async def play(track): + await asyncio.sleep(3600) + print(f"🎵 Finished: {track}") + + async def album(name, tracks): + async with asyncio.TaskGroup() as tg: + for track in tracks: + tg.create_task(play(track), name=track) + + async def main(): + print(f"PID: {os.getpid()}") + async with asyncio.TaskGroup() as tg: + tg.create_task( + album("Sundowning", ["TNDNBTG", "Levitate"]), + name="Sundowning", + ) + tg.create_task( + album("TMBTE", ["DYWTYLM", "Aqua Regia"]), + name="TMBTE", + ) + + asyncio.run(main()) + +Run the program in one terminal and leave it running: + +.. code-block:: shell-session + + $ python example.py + PID: 12345 + +Then pass the printed process ID to the commands from another terminal. +Thread IDs, task IDs, file paths, and line numbers vary between runs and +source layouts. + +.. versionadded:: 3.14 + +Command-line options +==================== + +.. option:: pstree PID + + Display task and coroutine relationships as a tree. Each task is shown + with its full coroutine stack, nested under the task (if any) that is + awaiting it. This subcommand is useful for quickly identifying which branch + of a task hierarchy is blocked and where in its coroutine stack execution + has paused: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + └── (T) Task-1 + └── main example.py:12 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) Sundowning + │ └── album example.py:7 + │ └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + │ └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + │ ├── (T) TNDNBTG + │ │ └── play example.py:4 + │ │ └── sleep Lib/asyncio/tasks.py:702 + │ └── (T) Levitate + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) TMBTE + └── album example.py:7 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) DYWTYLM + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) Aqua Regia + └── play example.py:4 + └── sleep Lib/asyncio/tasks.py:702 + + If the await graph contains a cycle, ``pstree`` reports an error instead + of printing a tree. A cycle in the await graph is unusual and typically + indicates a programming error: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + ERROR: await-graph contains cycles - cannot print a tree! + + cycle: Task-2 → Task-3 → Task-2 + +.. option:: ps PID + + Display a flat table of all pending tasks in the process *PID*. Each row + shows the event-loop thread ID, task ID and name, coroutine stack, and the + awaiting task's stack, name, and ID, if any. + + This subcommand prints all tasks regardless of whether the await graph + contains cycles: + + .. code-block:: shell-session + + $ python -m asyncio ps 12345 + tid task id task name coroutine stack awaiter chain awaiter name awaiter id + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 18445801 0x10a456060 Task-1 TaskGroup._aexit -> TaskGroup.__aexit__ -> main 0x0 + 18445801 0x10a439f60 Sundowning TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a439d70 TMBTE TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a2a3a80 TNDNBTG sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2a38a0 Levitate sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2d7150 DYWTYLM sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 + 18445801 0x10a6bdaa0 Aqua Regia sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 + +.. option:: --retries N + + Retry failed attempts to inspect the target process up to *N* times. This + can help when the target process changes while its state is being read. + + .. versionadded:: 3.15 diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 4ae6d1e43f2ac7..956b00f0873a0d 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -47,6 +47,13 @@ asyncio provides a set of **high-level** APIs to: * :ref:`synchronize <asyncio-sync>` concurrent code; +For **introspection**, asyncio provides APIs and tools for: + +* inspecting the :ref:`async call graph <asyncio-graph>` of tasks and futures; + +* inspecting tasks in another running Python process with + :ref:`command-line tools <asyncio-introspection-tools>`; + Additionally, there are **low-level** APIs for *library and framework developers* to: @@ -108,7 +115,13 @@ for full functionality and the latest features. asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst + +.. toctree:: + :caption: Introspection APIs + :maxdepth: 1 + asyncio-graph.rst + asyncio-tools.rst .. toctree:: :caption: Low-level APIs

1 0

[3.15] gh-119592: gh-152967: Fix ProcessPoolExecutor stranding submitted work when a max_tasks_per_child worker exits (GH-152978) (#153363)
by gpshead July 8, 2026

July 8, 2026

https://github.com/python/cpython/commit/cbbccf37b2be14fca026e087587b9611da… commit: cbbccf37b2be14fca026e087587b9611da371cbc branch: 3.15 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: gpshead <68491+gpshead(a)users.noreply.github.com> date: 2026-07-08T15:43:48-07:00 summary: [3.15] gh-119592: gh-152967: Fix ProcessPoolExecutor stranding submitted work when a max_tasks_per_child worker exits (GH-152978) (#153363) gh-119592: gh-152967: Fix ProcessPoolExecutor stranding submitted work when a max_tasks_per_child worker exits (GH-152978) gh-119592: Fix ProcessPoolExecutor stranding submitted work when a max_tasks_per_child worker exits Worker replacement went through the executor object: the manager thread read executor attributes that shutdown(wait=False) clears concurrently, and could not replace workers at all once the executor was garbage collected. A worker exiting at its max_tasks_per_child limit in those states left the remaining submitted work permanently unexecuted and hung interpreter exit; the racing case could crash the manager thread. Replace workers from the executor manager thread using its own state plus configuration read through the live executor weakref, which shutdown() never clears: - After shutdown(wait=False) with the executor still referenced, a replacement is spawned and the remaining work is executed as documented. - Once the executor has been garbage collected (gh-152967), or a replacement worker cannot be started and no workers remain, the remaining futures now fail with BrokenProcessPool instead of never resolving. - A new _force_shutting_down flag stops both spawn paths from starting workers that would escape terminate_workers()/kill_workers(). (cherry picked from commit 0c6422ff6a13ae309493fb7a358cb35d7ea959c8) Reviewed-multiple-times-by: Gregory P. Smith Co-authored-by: Gregory P. Smith <68491+gpshead(a)users.noreply.github.com> Co-authored-by: Claude Fable 5 <noreply(a)anthropic.com> files: A Misc/NEWS.d/next/Library/2026-07-03-18-30-00.gh-issue-119592.mQr3Vx.rst M Lib/concurrent/futures/process.py M Lib/test/test_concurrent_futures/test_process_pool.py diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 8f200fc1c82613f..e3b6c4a5305615a 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -269,6 +269,20 @@ def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=N return +def _spawn_worker(mp_context, call_queue, result_queue, initializer, + initargs, max_tasks_per_child, processes): + """Start one worker process and record it in *processes* by pid.""" + p = mp_context.Process( + target=_process_worker, + args=(call_queue, + result_queue, + initializer, + initargs, + max_tasks_per_child)) + p.start() + processes[p.pid] = p + + class _ExecutorManagerThread(threading.Thread): """Manages the communication between this process and the worker processes. @@ -321,6 +335,15 @@ def weakref_cb(_, # exiting safely self.max_tasks_per_child = executor._max_tasks_per_child + # gh-119592: Needed to size worker replacement, and immutable, so + # keep a copy rather than reading it back through the executor + # weakref. The rest of the spawn configuration is deliberately NOT + # copied here: holding user-provided objects (initializer, + # initargs, mp_context) in this always-reachable running thread + # could keep the executor itself reachable through them, breaking + # garbage-collection-triggered shutdown. + self.max_workers = executor._max_workers + # A dict mapping work ids to _WorkItems e.g. # {5: <_WorkItem...>, 6: <_WorkItem...>, ...} self.pending_work_items = executor._pending_work_items @@ -357,12 +380,14 @@ def run(self): # while waiting on new results. del result_item - if executor := self.executor_reference(): - if process_exited: - with self.shutdown_lock: - executor._replace_dead_worker() - else: - executor._idle_worker_semaphore.release() + if process_exited: + with self.shutdown_lock: + broken = self._replace_dead_worker() + if broken is not None: + self.terminate_broken(*broken) + return + elif executor := self.executor_reference(): + executor._idle_worker_semaphore.release() del executor if self.is_shutting_down(): @@ -379,6 +404,71 @@ def run(self): self.join_executor_internals() return + def _replace_dead_worker(self): + """Spawn a replacement for a worker that exited at its + max_tasks_per_child limit. Called under self.shutdown_lock. + + Returns None while the pool can still make progress, otherwise a + (cause, message) tuple describing why the remaining work items can + never run, so that run() can fail their futures. + """ + assert self.shutdown_lock.locked() + cause = None + message = None + executor = self.executor_reference() + if executor is None: + # gh-152967: The executor was garbage collected; nothing can + # spawn a replacement worker for it anymore. + message = ("The ProcessPoolExecutor was garbage collected with " + "work pending after its last worker process exited " + "upon reaching max_tasks_per_child; the pending work " + "can never be run.") + elif executor._force_shutting_down: + # terminate_workers()/kill_workers() is tearing the pool down; + # a replacement worker would escape the kill and run work + # items that were enqueued before it. + message = ("A worker process exited while the pool was being " + "forcefully shut down; work that was still enqueued " + "will not be run.") + elif self.pending_work_items or not self.is_shutting_down(): + # gh-115634: Do not consult the executor's + # _idle_worker_semaphore here: it counts task completions, not + # idle workers, so it can hold a stale token released by the + # now-dead worker. Trusting such a token would leave the pool + # a worker short, deadlocking once all workers reach their + # task limit. Spawning from this (manager) thread is safe + # despite gh-90622 because max_tasks_per_child is rejected for + # the "fork" start method. + if len(self.processes) < self.max_workers: + # gh-119592: Spawn using state owned by this thread and + # configuration read through the live weakref (which + # shutdown() never clears), not the executor state that + # shutdown(wait=False) clears concurrently. + try: + _spawn_worker(executor._mp_context, self.call_queue, + self.result_queue, executor._initializer, + executor._initargs, + self.max_tasks_per_child, self.processes) + except Exception as exc: + # While other workers remain the pool has merely lost + # capacity and they keep draining the queue; with none + # left the failure is reported below. + cause = format_exception(exc) + message = ("A replacement worker process could not be " + "started, leaving the pool without workers " + "to run the remaining work.") + del executor + + if not self.processes and (self.pending_work_items + or cause is not None): + # No worker processes remain and no replacement can be + # spawned: any remaining work items can never run. A spawn + # failure breaks the pool even with nothing pending; leaving + # a zero-worker pool alive would hang a later submit() on a + # stale _idle_worker_semaphore token instead of raising. + return (cause, message) + return None + def add_call_item_to_queue(self): # Fills call_queue with _WorkItems from pending_work_items. # This function never blocks. @@ -455,10 +545,11 @@ def is_shutting_down(self): return (_global_shutdown or executor is None or executor._shutdown_thread) - def _terminate_broken(self, cause): + def _terminate_broken(self, cause, bpe_message=None): # Terminate the executor because it is in a broken state. The cause # argument can be used to display more information on the error that - # lead the executor into becoming broken. + # lead the executor into becoming broken. bpe_message overrides the + # default message on the BrokenProcessPool set on pending futures. # Mark the process pool broken so that submits fail right now. executor = self.executor_reference() @@ -489,11 +580,12 @@ def _terminate_broken(self, cause): cause_str = "\n".join(errors) cause_tb = f"\n'''\n{cause_str}'''" if cause_str else None + if bpe_message is None: + bpe_message = ("A process in the process pool was terminated " + "abruptly while the future was running or pending.") # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): - bpe = BrokenProcessPool("A process in the process pool was " - "terminated abruptly while the future was " - "running or pending.") + bpe = BrokenProcessPool(bpe_message) if cause_tb is not None: bpe.__cause__ = _RemoteTraceback(cause_tb) try: @@ -518,9 +610,9 @@ def _terminate_broken(self, cause): # clean up resources self._join_executor_internals(broken=True) - def terminate_broken(self, cause): + def terminate_broken(self, cause, bpe_message=None): with self.shutdown_lock: - self._terminate_broken(cause) + self._terminate_broken(cause, bpe_message) def flag_executor_shutting_down(self): # Flag the executor as shutting down and cancel remaining tasks if @@ -733,6 +825,7 @@ def __init__(self, max_workers=None, mp_context=None, self._queue_count = 0 self._pending_work_items = {} self._cancel_pending_futures = False + self._force_shutting_down = False # _ThreadWakeup is a communication channel used to interrupt the wait # of the main loop of executor_manager_thread from another thread (e.g. @@ -772,34 +865,15 @@ def _start_executor_manager_thread(self): _threads_wakeups[self._executor_manager_thread] = \ self._executor_manager_thread_wakeup - def _replace_dead_worker(self): + def _adjust_process_count(self): # gh-132969: avoid error when state is reset and executor is still running, # which will happen when shutdown(wait=False) is called. if self._processes is None: return - # A replacement is pointless when shutting down with nothing left - # to run. Both attributes are read under _shutdown_lock, which - # shutdown() holds while setting _shutdown_thread. - assert self._shutdown_lock.locked() - if self._shutdown_thread and not self._pending_work_items: - return - - # gh-115634: A worker exited after reaching max_tasks_per_child and - # has been removed from self._processes. Do not consult - # _idle_worker_semaphore here: it counts task completions, not idle - # workers, so it can hold a stale token released by the now-dead - # worker. Trusting such a token would leave the pool a worker short, - # deadlocking once all workers reach their task limit. Spawning is - # safe from this (manager) thread despite gh-90622 because - # max_tasks_per_child is rejected for the "fork" start method. - if len(self._processes) < self._max_workers: - self._spawn_process() - - def _adjust_process_count(self): - # gh-132969: avoid error when state is reset and executor is still running, - # which will happen when shutdown(wait=False) is called. - if self._processes is None: + # gh-152967: A forceful shutdown is in progress; a worker spawned + # here could escape its process snapshot and keep running work. + if self._force_shutting_down: return # if there's an idle process, we don't need to spawn a new one. @@ -825,15 +899,10 @@ def _launch_processes(self): self._spawn_process() def _spawn_process(self): - p = self._mp_context.Process( - target=_process_worker, - args=(self._call_queue, - self._result_queue, - self._initializer, - self._initargs, - self._max_tasks_per_child)) - p.start() - self._processes[p.pid] = p + _spawn_worker(self._mp_context, self._call_queue, + self._result_queue, self._initializer, + self._initargs, self._max_tasks_per_child, + self._processes) def submit(self, fn, /, *args, **kwargs): with self._shutdown_lock: @@ -930,6 +999,14 @@ def _force_shutdown(self, operation): if operation not in _SHUTDOWN_CALLBACK_OPERATION: raise ValueError(f"Unsupported operation: {operation!r}") + # gh-152967: Stop the manager thread from spawning replacement + # workers before we copy the processes to signal: a worker spawned + # after the copy would survive the loop below and run enqueued + # work items. Taking the lock orders this against the manager's + # worker replacement, which runs under the same lock. + with self._shutdown_lock: + self._force_shutting_down = True + processes = {} if self._processes: processes = self._processes.copy() diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 38e2cb302c6d51f..c7d489c4f5b0a40 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -6,11 +6,12 @@ import traceback import unittest import unittest.mock +import weakref from concurrent import futures from concurrent.futures.process import BrokenProcessPool from test import support -from test.support import hashlib_helper, warnings_helper +from test.support import hashlib_helper, threading_helper, warnings_helper from test.test_importlib.metadata.fixtures import parameterize from .executor import ExecutorTest, mul @@ -42,6 +43,13 @@ def _put_wait_put(queue, event): queue.put('finished') +def _report_wait_return(queue, event, value): + """ Used as part of _run_stranded_worker_exit_test """ + queue.put(value) + event.wait() + return value + + class ProcessPoolExecutorTest(ExecutorTest): @unittest.skipUnless(sys.platform=='win32', 'Windows-only process limit') @@ -267,6 +275,126 @@ def test_max_tasks_per_child_pending_tasks_gh115634(self): finally: executor.shutdown(wait=True, cancel_futures=True) + def _run_stranded_worker_exit_test(self, *, shutdown, drop_reference): + # A worker exits upon reaching its max_tasks_per_child limit while + # more submitted work is queued. While the executor object is + # alive a replacement worker must be spawned and the remaining + # work executed; once it has been garbage collected no replacement + # is possible and the remaining futures must fail promptly instead + # of never resolving. + context = self.get_context() + if context.get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") + manager = context.Manager() + self.addCleanup(manager.join) + self.addCleanup(manager.shutdown) + started = manager.Queue() + gate = manager.Event() + + executor = self.executor_type( + 1, mp_context=context, max_tasks_per_child=1) + futs = [executor.submit(_report_wait_return, started, gate, i) + for i in range(3)] + self.addCleanup(threading_helper.join_thread, + executor._executor_manager_thread) + # Wait until the worker is inside the first task so that it exits + # at its task limit only after the executor has been shut down + # and/or garbage collected below. + self.assertEqual(started.get(timeout=support.SHORT_TIMEOUT), 0) + if shutdown: + executor.shutdown(wait=False) + if drop_reference: + executor_ref = weakref.ref(executor) + executor = None + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + support.gc_collect() + if executor_ref() is None: + break + gate.set() + + self.assertEqual(futs[0].result(timeout=support.SHORT_TIMEOUT), 0) + if drop_reference: + for fut in futs[1:]: + with self.assertRaisesRegex(BrokenProcessPool, + "garbage collected"): + fut.result(timeout=support.SHORT_TIMEOUT) + else: + results = [f.result(timeout=support.SHORT_TIMEOUT) + for f in futs[1:]] + self.assertEqual(results, [1, 2]) + + def test_shutdown_no_wait_max_tasks_gh119592(self): + # gh-119592: shutdown(wait=False) used to clear executor state that + # worker replacement relied on. A worker exiting at its + # max_tasks_per_child limit afterwards could not be replaced, so + # the remaining submitted work never ran, and a racing worker exit + # could crash the manager thread on the partially cleared state. + for drop_reference in (False, True): + with self.subTest(drop_reference=drop_reference): + self._run_stranded_worker_exit_test( + shutdown=True, drop_reference=drop_reference) + + def test_gc_during_max_tasks_worker_exit_gh152967(self): + # gh-152967: If the executor was garbage collected without + # shutdown() while its last worker exited at its + # max_tasks_per_child limit, no replacement worker could be + # spawned and the remaining futures were never resolved. + self._run_stranded_worker_exit_test( + shutdown=False, drop_reference=True) + + def _run_unreplaceable_worker_exit_test(self, *, error_regex, + force_shutting_down=False, + failing_spawn=False): + # Drive a max_tasks_per_child worker exit while worker + # replacement is impossible; the queued futures must fail + # promptly with a BrokenProcessPool explaining why. + context = self.get_context() + if context.get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") + manager = context.Manager() + self.addCleanup(manager.join) + self.addCleanup(manager.shutdown) + started = manager.Queue() + gate = manager.Event() + + executor = self.executor_type( + 1, mp_context=context, max_tasks_per_child=1) + futs = [executor.submit(_report_wait_return, started, gate, i) + for i in range(3)] + self.addCleanup(threading_helper.join_thread, + executor._executor_manager_thread) + self.assertEqual(started.get(timeout=support.SHORT_TIMEOUT), 0) + if force_shutting_down: + with executor._shutdown_lock: + executor._force_shutting_down = True + if failing_spawn: + spawn_patch = unittest.mock.patch( + "concurrent.futures.process._spawn_worker", + side_effect=OSError("spawn failed")) + spawn_patch.start() + self.addCleanup(spawn_patch.stop) + gate.set() + + self.assertEqual(futs[0].result(timeout=support.SHORT_TIMEOUT), 0) + for fut in futs[1:]: + with self.assertRaisesRegex(BrokenProcessPool, error_regex): + fut.result(timeout=support.SHORT_TIMEOUT) + + def test_force_shutdown_during_max_tasks_worker_exit(self): + # A worker exiting at its max_tasks_per_child limit during + # terminate_workers()/kill_workers() must not be replaced (the + # replacement would escape the kill); queued futures fail instead. + self._run_unreplaceable_worker_exit_test( + force_shutting_down=True, + error_regex="forcefully shut down") + + def test_failed_worker_replacement_breaks_pool(self): + # If no replacement worker can be started and no workers remain, + # the pool must break rather than strand the queued futures. + self._run_unreplaceable_worker_exit_test( + failing_spawn=True, + error_regex="could not be started") + def test_max_tasks_early_shutdown(self): context = self.get_context() if context.get_start_method(allow_none=False) == "fork": diff --git a/Misc/NEWS.d/next/Library/2026-07-03-18-30-00.gh-issue-119592.mQr3Vx.rst b/Misc/NEWS.d/next/Library/2026-07-03-18-30-00.gh-issue-119592.mQr3Vx.rst new file mode 100644 index 000000000000000..f718c877c8ed22b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-03-18-30-00.gh-issue-119592.mQr3Vx.rst @@ -0,0 +1,11 @@ +Fix :class:`concurrent.futures.ProcessPoolExecutor` stranding submitted +work forever when a worker process exited upon reaching its +*max_tasks_per_child* limit after +:meth:`~concurrent.futures.Executor.shutdown` was called with +``wait=False``: a replacement worker is now spawned and the remaining work +executed as documented. If the executor has instead been garbage collected +without ``shutdown()`` (:gh:`152967`), or a replacement worker cannot be +started, the remaining futures now fail with +:exc:`~concurrent.futures.process.BrokenProcessPool` instead of never +resolving. A worker exit racing ``shutdown(wait=False)`` can also no +longer crash the executor management thread.

1 0

[3.14] gh-153056: Fix a data race compiling the string.Template pattern in free-threading builds (GH-153057) (#153303)
by warsaw July 8, 2026

July 8, 2026

https://github.com/python/cpython/commit/6107e915526f9bf162452e98ce043c2df0… commit: 6107e915526f9bf162452e98ce043c2df0db712c branch: 3.14 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: warsaw <barry(a)python.org> date: 2026-07-08T14:43:54-07:00 summary: [3.14] gh-153056: Fix a data race compiling the string.Template pattern in free-threading builds (GH-153057) (#153303) gh-153056: Fix a data race compiling the string.Template pattern in free-threading builds (GH-153057) * gh-153056: Fix a data race compiling the string.Template pattern in free-threading builds Template compiles its substitution pattern lazily and caches it on the class. On the free-threaded build two concurrent first uses could race: a thread that observed the pattern another thread had just compiled would try to recompile it, and re.compile() rejects flags on an already-compiled pattern, raising a spurious ValueError. Return the already-compiled pattern instead. As a side effect, a subclass that supplies an already-compiled pattern now works too; previously it raised the same ValueError at class definition. * Trim test comments and NEWS wording * Document that the pattern attribute accepts a string or a compiled regex * Comment the three states of pattern and note the documented-behavior fix in NEWS * Update Doc/library/string.rst --------- (cherry picked from commit 45729033bff28f8abc36c42e802cb2853c205737) Co-authored-by: tonghuaroot (童话) <tonghuaroot(a)gmail.com> Co-authored-by: Barry Warsaw <barry(a)python.org> files: A Lib/test/test_free_threading/test_string_template_race.py A Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst M Doc/library/string.rst M Lib/string/__init__.py M Lib/test/test_string/test_string.py diff --git a/Doc/library/string.rst b/Doc/library/string.rst index be968a3c53d8430..f1abed91d20ed51 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -971,7 +971,8 @@ attributes: Alternatively, you can provide the entire regular expression pattern by overriding the class attribute *pattern*. If you do this, the value must be a -regular expression object with four named capturing groups. The capturing +regular expression pattern string, or a compiled regular expression +object, with four named capturing groups. The capturing groups correspond to the rules given above, along with the invalid placeholder rule: diff --git a/Lib/string/__init__.py b/Lib/string/__init__.py index eab5067c9b133ea..b2c369ef9d1f76c 100644 --- a/Lib/string/__init__.py +++ b/Lib/string/__init__.py @@ -83,7 +83,14 @@ def __init_subclass__(cls): def _compile_pattern(cls): import re # deferred import, for performance + # `pattern` may be the `_TemplatePattern` sentinel (not yet compiled), an + # already-compiled regular expression object (as documented), or a string + # regular expression. An already-compiled object is returned as-is; the + # other two are compiled and cached back on the class. pattern = cls.__dict__.get('pattern', _TemplatePattern) + if isinstance(pattern, re.Pattern): + # re.compile() rejects flags on an already-compiled pattern. + return pattern if pattern is _TemplatePattern: delim = re.escape(cls.delimiter) id = cls.idpattern diff --git a/Lib/test/test_free_threading/test_string_template_race.py b/Lib/test/test_free_threading/test_string_template_race.py new file mode 100644 index 000000000000000..dc4b4e9f18d2863 --- /dev/null +++ b/Lib/test/test_free_threading/test_string_template_race.py @@ -0,0 +1,35 @@ +import string +import unittest +from string import Template + +from test.support import threading_helper + + +(a)threading_helper.requires_working_threading() +class TestTemplateCompileRace(unittest.TestCase): + def test_concurrent_first_use(self): + # Racing the lazy pattern compile must not raise a spurious ValueError + # from recompiling an already-compiled pattern. A throwaway subclass, + # re-armed to the sentinel each round, keeps string.Template unmutated + # (subclasses precompile eagerly in __init_subclass__). + uncompiled = string._TemplatePattern + errors = [] + + def use_template(cls): + try: + cls("$x and ${y}").substitute(x=1, y=2) + except Exception as e: + errors.append(e) + + for _ in range(20): + class T(Template): + pass + T.pattern = uncompiled + T.flags = None + threading_helper.run_concurrently(use_template, nthreads=10, args=(T,)) + + self.assertEqual(errors, [], msg=f"unexpected errors: {errors}") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_string/test_string.py b/Lib/test/test_string/test_string.py index 5394fe4e12cd41c..350784a9f00817b 100644 --- a/Lib/test/test_string/test_string.py +++ b/Lib/test/test_string/test_string.py @@ -299,6 +299,20 @@ def test_SafeTemplate(self): eq(s.safe_substitute(dict(who='tim', what='ham', meal='dinner')), 'tim likes ham for dinner') + def test_precompiled_pattern(self): + # A subclass may supply an already-compiled pattern; it must be reused, + # not recompiled (re.compile() rejects flags on a compiled pattern). + import re + compiled = re.compile( + r'\$(?:(?P<escaped>\$)|(?P<named>[a-z]+)|' + r'\{(?P<braced>[a-z]+)\}|(?P<invalid>))') + class MyTemplate(Template): + pattern = compiled + self.assertIs(MyTemplate.pattern, compiled) + self.assertEqual( + MyTemplate('$who likes $what').substitute(who='tim', what='ham'), + 'tim likes ham') + def test_invalid_placeholders(self): raises = self.assertRaises s = Template('$who likes $') diff --git a/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst b/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst new file mode 100644 index 000000000000000..a2bc89f25a53c19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst @@ -0,0 +1,4 @@ +Fix :class:`string.Template` raising a spurious :exc:`ValueError` when the +*pattern* attribute is a compiled regular expression object, which the +documentation allows. On the free-threaded build this also occurred as a data +race on the first concurrent use.

1 0

gh-119592: gh-152967: Fix ProcessPoolExecutor stranding submitted work when a max_tasks_per_child worker exits (GH-152978)
by gpshead July 8, 2026

July 8, 2026

https://github.com/python/cpython/commit/0c6422ff6a13ae309493fb7a358cb35d7e… commit: 0c6422ff6a13ae309493fb7a358cb35d7ea959c8 branch: main author: Gregory P. Smith <68491+gpshead(a)users.noreply.github.com> committer: gpshead <68491+gpshead(a)users.noreply.github.com> date: 2026-07-08T13:06:20-07:00 summary: gh-119592: gh-152967: Fix ProcessPoolExecutor stranding submitted work when a max_tasks_per_child worker exits (GH-152978) gh-119592: Fix ProcessPoolExecutor stranding submitted work when a max_tasks_per_child worker exits Worker replacement went through the executor object: the manager thread read executor attributes that shutdown(wait=False) clears concurrently, and could not replace workers at all once the executor was garbage collected. A worker exiting at its max_tasks_per_child limit in those states left the remaining submitted work permanently unexecuted and hung interpreter exit; the racing case could crash the manager thread. Replace workers from the executor manager thread using its own state plus configuration read through the live executor weakref, which shutdown() never clears: - After shutdown(wait=False) with the executor still referenced, a replacement is spawned and the remaining work is executed as documented. - Once the executor has been garbage collected (gh-152967), or a replacement worker cannot be started and no workers remain, the remaining futures now fail with BrokenProcessPool instead of never resolving. - A new _force_shutting_down flag stops both spawn paths from starting workers that would escape terminate_workers()/kill_workers(). Co-authored-by: Claude Fable 5 <noreply(a)anthropic.com> Reviewed-multiple-times-by: Gregory P. Smith files: A Misc/NEWS.d/next/Library/2026-07-03-18-30-00.gh-issue-119592.mQr3Vx.rst M Lib/concurrent/futures/process.py M Lib/test/test_concurrent_futures/test_process_pool.py diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 8f200fc1c82613f..e3b6c4a5305615a 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -269,6 +269,20 @@ def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=N return +def _spawn_worker(mp_context, call_queue, result_queue, initializer, + initargs, max_tasks_per_child, processes): + """Start one worker process and record it in *processes* by pid.""" + p = mp_context.Process( + target=_process_worker, + args=(call_queue, + result_queue, + initializer, + initargs, + max_tasks_per_child)) + p.start() + processes[p.pid] = p + + class _ExecutorManagerThread(threading.Thread): """Manages the communication between this process and the worker processes. @@ -321,6 +335,15 @@ def weakref_cb(_, # exiting safely self.max_tasks_per_child = executor._max_tasks_per_child + # gh-119592: Needed to size worker replacement, and immutable, so + # keep a copy rather than reading it back through the executor + # weakref. The rest of the spawn configuration is deliberately NOT + # copied here: holding user-provided objects (initializer, + # initargs, mp_context) in this always-reachable running thread + # could keep the executor itself reachable through them, breaking + # garbage-collection-triggered shutdown. + self.max_workers = executor._max_workers + # A dict mapping work ids to _WorkItems e.g. # {5: <_WorkItem...>, 6: <_WorkItem...>, ...} self.pending_work_items = executor._pending_work_items @@ -357,12 +380,14 @@ def run(self): # while waiting on new results. del result_item - if executor := self.executor_reference(): - if process_exited: - with self.shutdown_lock: - executor._replace_dead_worker() - else: - executor._idle_worker_semaphore.release() + if process_exited: + with self.shutdown_lock: + broken = self._replace_dead_worker() + if broken is not None: + self.terminate_broken(*broken) + return + elif executor := self.executor_reference(): + executor._idle_worker_semaphore.release() del executor if self.is_shutting_down(): @@ -379,6 +404,71 @@ def run(self): self.join_executor_internals() return + def _replace_dead_worker(self): + """Spawn a replacement for a worker that exited at its + max_tasks_per_child limit. Called under self.shutdown_lock. + + Returns None while the pool can still make progress, otherwise a + (cause, message) tuple describing why the remaining work items can + never run, so that run() can fail their futures. + """ + assert self.shutdown_lock.locked() + cause = None + message = None + executor = self.executor_reference() + if executor is None: + # gh-152967: The executor was garbage collected; nothing can + # spawn a replacement worker for it anymore. + message = ("The ProcessPoolExecutor was garbage collected with " + "work pending after its last worker process exited " + "upon reaching max_tasks_per_child; the pending work " + "can never be run.") + elif executor._force_shutting_down: + # terminate_workers()/kill_workers() is tearing the pool down; + # a replacement worker would escape the kill and run work + # items that were enqueued before it. + message = ("A worker process exited while the pool was being " + "forcefully shut down; work that was still enqueued " + "will not be run.") + elif self.pending_work_items or not self.is_shutting_down(): + # gh-115634: Do not consult the executor's + # _idle_worker_semaphore here: it counts task completions, not + # idle workers, so it can hold a stale token released by the + # now-dead worker. Trusting such a token would leave the pool + # a worker short, deadlocking once all workers reach their + # task limit. Spawning from this (manager) thread is safe + # despite gh-90622 because max_tasks_per_child is rejected for + # the "fork" start method. + if len(self.processes) < self.max_workers: + # gh-119592: Spawn using state owned by this thread and + # configuration read through the live weakref (which + # shutdown() never clears), not the executor state that + # shutdown(wait=False) clears concurrently. + try: + _spawn_worker(executor._mp_context, self.call_queue, + self.result_queue, executor._initializer, + executor._initargs, + self.max_tasks_per_child, self.processes) + except Exception as exc: + # While other workers remain the pool has merely lost + # capacity and they keep draining the queue; with none + # left the failure is reported below. + cause = format_exception(exc) + message = ("A replacement worker process could not be " + "started, leaving the pool without workers " + "to run the remaining work.") + del executor + + if not self.processes and (self.pending_work_items + or cause is not None): + # No worker processes remain and no replacement can be + # spawned: any remaining work items can never run. A spawn + # failure breaks the pool even with nothing pending; leaving + # a zero-worker pool alive would hang a later submit() on a + # stale _idle_worker_semaphore token instead of raising. + return (cause, message) + return None + def add_call_item_to_queue(self): # Fills call_queue with _WorkItems from pending_work_items. # This function never blocks. @@ -455,10 +545,11 @@ def is_shutting_down(self): return (_global_shutdown or executor is None or executor._shutdown_thread) - def _terminate_broken(self, cause): + def _terminate_broken(self, cause, bpe_message=None): # Terminate the executor because it is in a broken state. The cause # argument can be used to display more information on the error that - # lead the executor into becoming broken. + # lead the executor into becoming broken. bpe_message overrides the + # default message on the BrokenProcessPool set on pending futures. # Mark the process pool broken so that submits fail right now. executor = self.executor_reference() @@ -489,11 +580,12 @@ def _terminate_broken(self, cause): cause_str = "\n".join(errors) cause_tb = f"\n'''\n{cause_str}'''" if cause_str else None + if bpe_message is None: + bpe_message = ("A process in the process pool was terminated " + "abruptly while the future was running or pending.") # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): - bpe = BrokenProcessPool("A process in the process pool was " - "terminated abruptly while the future was " - "running or pending.") + bpe = BrokenProcessPool(bpe_message) if cause_tb is not None: bpe.__cause__ = _RemoteTraceback(cause_tb) try: @@ -518,9 +610,9 @@ def _terminate_broken(self, cause): # clean up resources self._join_executor_internals(broken=True) - def terminate_broken(self, cause): + def terminate_broken(self, cause, bpe_message=None): with self.shutdown_lock: - self._terminate_broken(cause) + self._terminate_broken(cause, bpe_message) def flag_executor_shutting_down(self): # Flag the executor as shutting down and cancel remaining tasks if @@ -733,6 +825,7 @@ def __init__(self, max_workers=None, mp_context=None, self._queue_count = 0 self._pending_work_items = {} self._cancel_pending_futures = False + self._force_shutting_down = False # _ThreadWakeup is a communication channel used to interrupt the wait # of the main loop of executor_manager_thread from another thread (e.g. @@ -772,34 +865,15 @@ def _start_executor_manager_thread(self): _threads_wakeups[self._executor_manager_thread] = \ self._executor_manager_thread_wakeup - def _replace_dead_worker(self): + def _adjust_process_count(self): # gh-132969: avoid error when state is reset and executor is still running, # which will happen when shutdown(wait=False) is called. if self._processes is None: return - # A replacement is pointless when shutting down with nothing left - # to run. Both attributes are read under _shutdown_lock, which - # shutdown() holds while setting _shutdown_thread. - assert self._shutdown_lock.locked() - if self._shutdown_thread and not self._pending_work_items: - return - - # gh-115634: A worker exited after reaching max_tasks_per_child and - # has been removed from self._processes. Do not consult - # _idle_worker_semaphore here: it counts task completions, not idle - # workers, so it can hold a stale token released by the now-dead - # worker. Trusting such a token would leave the pool a worker short, - # deadlocking once all workers reach their task limit. Spawning is - # safe from this (manager) thread despite gh-90622 because - # max_tasks_per_child is rejected for the "fork" start method. - if len(self._processes) < self._max_workers: - self._spawn_process() - - def _adjust_process_count(self): - # gh-132969: avoid error when state is reset and executor is still running, - # which will happen when shutdown(wait=False) is called. - if self._processes is None: + # gh-152967: A forceful shutdown is in progress; a worker spawned + # here could escape its process snapshot and keep running work. + if self._force_shutting_down: return # if there's an idle process, we don't need to spawn a new one. @@ -825,15 +899,10 @@ def _launch_processes(self): self._spawn_process() def _spawn_process(self): - p = self._mp_context.Process( - target=_process_worker, - args=(self._call_queue, - self._result_queue, - self._initializer, - self._initargs, - self._max_tasks_per_child)) - p.start() - self._processes[p.pid] = p + _spawn_worker(self._mp_context, self._call_queue, + self._result_queue, self._initializer, + self._initargs, self._max_tasks_per_child, + self._processes) def submit(self, fn, /, *args, **kwargs): with self._shutdown_lock: @@ -930,6 +999,14 @@ def _force_shutdown(self, operation): if operation not in _SHUTDOWN_CALLBACK_OPERATION: raise ValueError(f"Unsupported operation: {operation!r}") + # gh-152967: Stop the manager thread from spawning replacement + # workers before we copy the processes to signal: a worker spawned + # after the copy would survive the loop below and run enqueued + # work items. Taking the lock orders this against the manager's + # worker replacement, which runs under the same lock. + with self._shutdown_lock: + self._force_shutting_down = True + processes = {} if self._processes: processes = self._processes.copy() diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 205662c91c2558d..dafbda862c51c24 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -6,11 +6,12 @@ import traceback import unittest import unittest.mock +import weakref from concurrent import futures from concurrent.futures.process import BrokenProcessPool from test import support -from test.support import hashlib_helper, warnings_helper +from test.support import hashlib_helper, threading_helper, warnings_helper from test.test_importlib.metadata.fixtures import parameterize from .executor import ExecutorTest, mul @@ -42,6 +43,13 @@ def _put_wait_put(queue, event): queue.put('finished') +def _report_wait_return(queue, event, value): + """ Used as part of _run_stranded_worker_exit_test """ + queue.put(value) + event.wait() + return value + + class ProcessPoolExecutorTest(ExecutorTest): @unittest.skipUnless(sys.platform=='win32', 'Windows-only process limit') @@ -268,6 +276,126 @@ def test_max_tasks_per_child_pending_tasks_gh115634(self): finally: executor.shutdown(wait=True, cancel_futures=True) + def _run_stranded_worker_exit_test(self, *, shutdown, drop_reference): + # A worker exits upon reaching its max_tasks_per_child limit while + # more submitted work is queued. While the executor object is + # alive a replacement worker must be spawned and the remaining + # work executed; once it has been garbage collected no replacement + # is possible and the remaining futures must fail promptly instead + # of never resolving. + context = self.get_context() + if context.get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") + manager = context.Manager() + self.addCleanup(manager.join) + self.addCleanup(manager.shutdown) + started = manager.Queue() + gate = manager.Event() + + executor = self.executor_type( + 1, mp_context=context, max_tasks_per_child=1) + futs = [executor.submit(_report_wait_return, started, gate, i) + for i in range(3)] + self.addCleanup(threading_helper.join_thread, + executor._executor_manager_thread) + # Wait until the worker is inside the first task so that it exits + # at its task limit only after the executor has been shut down + # and/or garbage collected below. + self.assertEqual(started.get(timeout=support.SHORT_TIMEOUT), 0) + if shutdown: + executor.shutdown(wait=False) + if drop_reference: + executor_ref = weakref.ref(executor) + executor = None + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + support.gc_collect() + if executor_ref() is None: + break + gate.set() + + self.assertEqual(futs[0].result(timeout=support.SHORT_TIMEOUT), 0) + if drop_reference: + for fut in futs[1:]: + with self.assertRaisesRegex(BrokenProcessPool, + "garbage collected"): + fut.result(timeout=support.SHORT_TIMEOUT) + else: + results = [f.result(timeout=support.SHORT_TIMEOUT) + for f in futs[1:]] + self.assertEqual(results, [1, 2]) + + def test_shutdown_no_wait_max_tasks_gh119592(self): + # gh-119592: shutdown(wait=False) used to clear executor state that + # worker replacement relied on. A worker exiting at its + # max_tasks_per_child limit afterwards could not be replaced, so + # the remaining submitted work never ran, and a racing worker exit + # could crash the manager thread on the partially cleared state. + for drop_reference in (False, True): + with self.subTest(drop_reference=drop_reference): + self._run_stranded_worker_exit_test( + shutdown=True, drop_reference=drop_reference) + + def test_gc_during_max_tasks_worker_exit_gh152967(self): + # gh-152967: If the executor was garbage collected without + # shutdown() while its last worker exited at its + # max_tasks_per_child limit, no replacement worker could be + # spawned and the remaining futures were never resolved. + self._run_stranded_worker_exit_test( + shutdown=False, drop_reference=True) + + def _run_unreplaceable_worker_exit_test(self, *, error_regex, + force_shutting_down=False, + failing_spawn=False): + # Drive a max_tasks_per_child worker exit while worker + # replacement is impossible; the queued futures must fail + # promptly with a BrokenProcessPool explaining why. + context = self.get_context() + if context.get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") + manager = context.Manager() + self.addCleanup(manager.join) + self.addCleanup(manager.shutdown) + started = manager.Queue() + gate = manager.Event() + + executor = self.executor_type( + 1, mp_context=context, max_tasks_per_child=1) + futs = [executor.submit(_report_wait_return, started, gate, i) + for i in range(3)] + self.addCleanup(threading_helper.join_thread, + executor._executor_manager_thread) + self.assertEqual(started.get(timeout=support.SHORT_TIMEOUT), 0) + if force_shutting_down: + with executor._shutdown_lock: + executor._force_shutting_down = True + if failing_spawn: + spawn_patch = unittest.mock.patch( + "concurrent.futures.process._spawn_worker", + side_effect=OSError("spawn failed")) + spawn_patch.start() + self.addCleanup(spawn_patch.stop) + gate.set() + + self.assertEqual(futs[0].result(timeout=support.SHORT_TIMEOUT), 0) + for fut in futs[1:]: + with self.assertRaisesRegex(BrokenProcessPool, error_regex): + fut.result(timeout=support.SHORT_TIMEOUT) + + def test_force_shutdown_during_max_tasks_worker_exit(self): + # A worker exiting at its max_tasks_per_child limit during + # terminate_workers()/kill_workers() must not be replaced (the + # replacement would escape the kill); queued futures fail instead. + self._run_unreplaceable_worker_exit_test( + force_shutting_down=True, + error_regex="forcefully shut down") + + def test_failed_worker_replacement_breaks_pool(self): + # If no replacement worker can be started and no workers remain, + # the pool must break rather than strand the queued futures. + self._run_unreplaceable_worker_exit_test( + failing_spawn=True, + error_regex="could not be started") + def test_max_tasks_early_shutdown(self): context = self.get_context() if context.get_start_method(allow_none=False) == "fork": diff --git a/Misc/NEWS.d/next/Library/2026-07-03-18-30-00.gh-issue-119592.mQr3Vx.rst b/Misc/NEWS.d/next/Library/2026-07-03-18-30-00.gh-issue-119592.mQr3Vx.rst new file mode 100644 index 000000000000000..f718c877c8ed22b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-03-18-30-00.gh-issue-119592.mQr3Vx.rst @@ -0,0 +1,11 @@ +Fix :class:`concurrent.futures.ProcessPoolExecutor` stranding submitted +work forever when a worker process exited upon reaching its +*max_tasks_per_child* limit after +:meth:`~concurrent.futures.Executor.shutdown` was called with +``wait=False``: a replacement worker is now spawned and the remaining work +executed as documented. If the executor has instead been garbage collected +without ``shutdown()`` (:gh:`152967`), or a replacement worker cannot be +started, the remaining futures now fail with +:exc:`~concurrent.futures.process.BrokenProcessPool` instead of never +resolving. A worker exit racing ``shutdown(wait=False)`` can also no +longer crash the executor management thread.

1 0

[3.15] gh-153300: Set global configuration variables in PyConfig_Set() (GH-153301) (#153356)
by vstinner July 8, 2026

July 8, 2026

https://github.com/python/cpython/commit/c5e73f8c91dd7a6aa7587b60e526b3e3a9… commit: c5e73f8c91dd7a6aa7587b60e526b3e3a915bb3b branch: 3.15 author: Miss Islington (bot) <31488909+miss-islington(a)users.noreply.github.com> committer: vstinner <vstinner(a)python.org> date: 2026-07-08T20:21:20+02:00 summary: [3.15] gh-153300: Set global configuration variables in PyConfig_Set() (GH-153301) (#153356) gh-153300: Set global configuration variables in PyConfig_Set() (GH-153301) PyConfig_Set() now also set global configuration variables. For example, PyConfig_Set("inspect", value) now also sets Py_InspectFlag. Use PyConfig_Set() in main.c to set the inspect flag. Python code can now see the updated sys.flags.inspect value. (cherry picked from commit 9fb713f1b8b774c789db007e3824c766a626ccff) Co-authored-by: Victor Stinner <vstinner(a)python.org> files: A Misc/NEWS.d/next/C_API/2026-07-08-00-38-32.gh-issue-153300.lVoWyR.rst M Lib/test/test_capi/test_config.py M Modules/main.c M Python/initconfig.c diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index c750a6c2a477abd..290126343618381 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -9,6 +9,7 @@ from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') # Is the Py_STATS macro defined? @@ -378,6 +379,46 @@ def expect_bool_not(value): finally: config_set(name, old_value) + def test_config_set_global_vars(self): + # Test PyConfig_Set() with global configuration variables + config_get = _testcapi.config_get + config_set = _testcapi.config_set + get_configs = _testinternalcapi.get_configs + new_values = (0, 1, 5) + + for name, global_name, not_value in ( + ('bytes_warning', 'Py_BytesWarningFlag', False), + ('inspect', 'Py_InspectFlag', False), + ('interactive', 'Py_InteractiveFlag', False), + ('optimization_level', 'Py_OptimizeFlag', False), + ('parser_debug', 'Py_DebugFlag', False), + ('quiet', 'Py_QuietFlag', False), + ('use_environment', 'Py_IgnoreEnvironmentFlag', True), + ('verbose', 'Py_VerboseFlag', False), + ('write_bytecode', 'Py_DontWriteBytecodeFlag', True), + + # Read-only variables + #('buffered_stdio', 'Py_UnbufferedStdioFlag', True), + #('isolated', 'Py_IsolatedFlag', False), + #('pathconfig_warnings', 'Py_FrozenFlag', True), + #('site_import', 'Py_NoSiteFlag', True), + #('user_site_directory', 'Py_NoUserSiteDirectory', True), + # Windows only + #('legacy_windows_stdio', 'Py_LegacyWindowsStdioFlag', False) + ): + with self.subTest(name=name): + old_value = config_get(name) + try: + for value in new_values: + config_set(name, value) + global_config = get_configs()['global_config'] + expected = value + if not_value: + expected = int(not value) + self.assertEqual(global_config[global_name], expected) + finally: + config_set(name, old_value) + def test_config_set_cpu_count(self): config_get = _testcapi.config_get config_set = _testcapi.config_set diff --git a/Misc/NEWS.d/next/C_API/2026-07-08-00-38-32.gh-issue-153300.lVoWyR.rst b/Misc/NEWS.d/next/C_API/2026-07-08-00-38-32.gh-issue-153300.lVoWyR.rst new file mode 100644 index 000000000000000..3c19c0e15d2bc65 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-07-08-00-38-32.gh-issue-153300.lVoWyR.rst @@ -0,0 +1,3 @@ +:c:func:`PyConfig_Set()` now also set global configuration variables. For +example, ``PyConfig_Set("inspect", value)`` now also sets +:c:var:`Py_InspectFlag`. Patch by Victor Stinner. diff --git a/Modules/main.c b/Modules/main.c index a4dfddd98e257e2..2f23513cc2403dc 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -529,11 +529,15 @@ pymain_run_interactive_hook(int *exitcode) static void pymain_set_inspect(PyConfig *config, int inspect) { - config->inspect = inspect; -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - Py_InspectFlag = inspect; -_Py_COMP_DIAG_POP + PyObject *value = PyLong_FromLong(inspect); + if (value == NULL || PyConfig_Set("inspect", value) < 0) { + fprintf(stderr, "Could not set the inspect flag\n"); + PyErr_Print(); + } + else { + assert(config->inspect == inspect); + } + Py_XDECREF(value); } @@ -635,7 +639,7 @@ pymain_run_python(int *exitcode) { PyObject *main_importer_path = NULL; PyInterpreterState *interp = _PyInterpreterState_GET(); - /* pymain_run_stdin() modify the config */ + /* pymain_repl() and pymain_run_stdin() modify the config */ PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp); /* ensure path config is written into global variables */ diff --git a/Python/initconfig.c b/Python/initconfig.c index 185b5b107d68dbd..a7cd08de8cc492e 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -84,120 +84,139 @@ typedef struct { config_sys_flag_setter flag_setter; } PyConfigSysSpec; +typedef struct { + int *ptr; + int not; +} PyConfigGlobalVar; + typedef struct { const char *name; size_t offset; PyConfigMemberType type; PyConfigMemberVisibility visibility; PyConfigSysSpec sys; + PyConfigGlobalVar global_var; } PyConfigSpec; -#define SPEC(MEMBER, TYPE, VISIBILITY, sys) \ +#define SPEC(MEMBER, TYPE, VISIBILITY, sys, global_var) \ {#MEMBER, offsetof(PyConfig, MEMBER), \ - PyConfig_MEMBER_##TYPE, PyConfig_MEMBER_##VISIBILITY, sys} + PyConfig_MEMBER_##TYPE, PyConfig_MEMBER_##VISIBILITY, sys, global_var} #define SYS_ATTR(name) {name, -1, NULL} #define SYS_FLAG_SETTER(index, setter) {NULL, index, setter} #define SYS_FLAG(index) SYS_FLAG_SETTER(index, NULL) #define NO_SYS SYS_ATTR(NULL) +#define GLOBAL(ptr, not) {ptr, not} +#define NO_GLOBAL GLOBAL(NULL, 0) + +// Ignore deprecations on global variables such as Py_IsolatedFlag +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS + // Update _test_embed_set_config when adding new members static const PyConfigSpec PYCONFIG_SPEC[] = { // --- Public options ----------- - SPEC(argv, WSTR_LIST, PUBLIC, SYS_ATTR("argv")), - SPEC(base_exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_exec_prefix")), - SPEC(base_executable, WSTR_OPT, PUBLIC, SYS_ATTR("_base_executable")), - SPEC(base_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_prefix")), - SPEC(bytes_warning, UINT, PUBLIC, SYS_FLAG(9)), - SPEC(cpu_count, INT, PUBLIC, NO_SYS), - SPEC(lazy_imports, INT, PUBLIC, NO_SYS), - SPEC(exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("exec_prefix")), - SPEC(executable, WSTR_OPT, PUBLIC, SYS_ATTR("executable")), - SPEC(inspect, BOOL, PUBLIC, SYS_FLAG(1)), - SPEC(int_max_str_digits, UINT, PUBLIC, NO_SYS), - SPEC(interactive, BOOL, PUBLIC, SYS_FLAG(2)), - SPEC(module_search_paths, WSTR_LIST, PUBLIC, SYS_ATTR("path")), - SPEC(optimization_level, UINT, PUBLIC, SYS_FLAG(3)), - SPEC(parser_debug, BOOL, PUBLIC, SYS_FLAG(0)), - SPEC(platlibdir, WSTR, PUBLIC, SYS_ATTR("platlibdir")), - SPEC(prefix, WSTR_OPT, PUBLIC, SYS_ATTR("prefix")), - SPEC(pycache_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("pycache_prefix")), - SPEC(quiet, BOOL, PUBLIC, SYS_FLAG(10)), - SPEC(stdlib_dir, WSTR_OPT, PUBLIC, SYS_ATTR("_stdlib_dir")), - SPEC(use_environment, BOOL, PUBLIC, SYS_FLAG_SETTER(7, config_sys_flag_not)), - SPEC(verbose, UINT, PUBLIC, SYS_FLAG(8)), - SPEC(warnoptions, WSTR_LIST, PUBLIC, SYS_ATTR("warnoptions")), - SPEC(write_bytecode, BOOL, PUBLIC, SYS_FLAG_SETTER(4, config_sys_flag_not)), - SPEC(xoptions, WSTR_LIST, PUBLIC, SYS_ATTR("_xoptions")), + SPEC(argv, WSTR_LIST, PUBLIC, SYS_ATTR("argv"), NO_GLOBAL), + SPEC(base_exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_exec_prefix"), NO_GLOBAL), + SPEC(base_executable, WSTR_OPT, PUBLIC, SYS_ATTR("_base_executable"), NO_GLOBAL), + SPEC(base_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_prefix"), NO_GLOBAL), + SPEC(bytes_warning, UINT, PUBLIC, SYS_FLAG(9), GLOBAL(&Py_BytesWarningFlag, 0)), + SPEC(cpu_count, INT, PUBLIC, NO_SYS, NO_GLOBAL), + SPEC(lazy_imports, INT, PUBLIC, NO_SYS, NO_GLOBAL), + SPEC(exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("exec_prefix"), NO_GLOBAL), + SPEC(executable, WSTR_OPT, PUBLIC, SYS_ATTR("executable"), NO_GLOBAL), + SPEC(inspect, BOOL, PUBLIC, SYS_FLAG(1), GLOBAL(&Py_InspectFlag, 0)), + SPEC(int_max_str_digits, UINT, PUBLIC, NO_SYS, NO_GLOBAL), + SPEC(interactive, BOOL, PUBLIC, SYS_FLAG(2), GLOBAL(&Py_InteractiveFlag, 0)), + SPEC(module_search_paths, WSTR_LIST, PUBLIC, SYS_ATTR("path"), NO_GLOBAL), + SPEC(optimization_level, UINT, PUBLIC, SYS_FLAG(3), GLOBAL(&Py_OptimizeFlag, 0)), + SPEC(parser_debug, BOOL, PUBLIC, SYS_FLAG(0), GLOBAL(&Py_DebugFlag, 0)), + SPEC(platlibdir, WSTR, PUBLIC, SYS_ATTR("platlibdir"), NO_GLOBAL), + SPEC(prefix, WSTR_OPT, PUBLIC, SYS_ATTR("prefix"), NO_GLOBAL), + SPEC(pycache_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("pycache_prefix"), NO_GLOBAL), + SPEC(quiet, BOOL, PUBLIC, SYS_FLAG(10), GLOBAL(&Py_QuietFlag, 0)), + SPEC(stdlib_dir, WSTR_OPT, PUBLIC, SYS_ATTR("_stdlib_dir"), NO_GLOBAL), + SPEC(use_environment, BOOL, PUBLIC, + SYS_FLAG_SETTER(7, config_sys_flag_not), GLOBAL(&Py_IgnoreEnvironmentFlag, 1)), + SPEC(verbose, UINT, PUBLIC, SYS_FLAG(8), GLOBAL(&Py_VerboseFlag, 0)), + SPEC(warnoptions, WSTR_LIST, PUBLIC, SYS_ATTR("warnoptions"), NO_GLOBAL), + SPEC(write_bytecode, BOOL, PUBLIC, SYS_FLAG_SETTER(4, config_sys_flag_not), + GLOBAL(&Py_DontWriteBytecodeFlag, 1)), + SPEC(xoptions, WSTR_LIST, PUBLIC, SYS_ATTR("_xoptions"), NO_GLOBAL), // --- Read-only options ----------- #ifdef Py_STATS - SPEC(_pystats, BOOL, READ_ONLY, NO_SYS), + SPEC(_pystats, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), #endif - SPEC(buffered_stdio, BOOL, READ_ONLY, NO_SYS), - SPEC(check_hash_pycs_mode, WSTR, READ_ONLY, NO_SYS), - SPEC(code_debug_ranges, BOOL, READ_ONLY, NO_SYS), - SPEC(configure_c_stdio, BOOL, READ_ONLY, NO_SYS), - SPEC(dev_mode, BOOL, READ_ONLY, NO_SYS), // sys.flags.dev_mode - SPEC(dump_refs, BOOL, READ_ONLY, NO_SYS), - SPEC(dump_refs_file, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(buffered_stdio, BOOL, READ_ONLY, NO_SYS, + GLOBAL(&Py_UnbufferedStdioFlag, 1)), + SPEC(check_hash_pycs_mode, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(code_debug_ranges, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(configure_c_stdio, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(dev_mode, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), // sys.flags.dev_mode + SPEC(dump_refs, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(dump_refs_file, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), #ifdef Py_GIL_DISABLED - SPEC(enable_gil, INT, READ_ONLY, NO_SYS), - SPEC(tlbc_enabled, INT, READ_ONLY, NO_SYS), + SPEC(enable_gil, INT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(tlbc_enabled, INT, READ_ONLY, NO_SYS, NO_GLOBAL), #endif - SPEC(faulthandler, BOOL, READ_ONLY, NO_SYS), - SPEC(filesystem_encoding, WSTR, READ_ONLY, NO_SYS), - SPEC(filesystem_errors, WSTR, READ_ONLY, NO_SYS), - SPEC(hash_seed, ULONG, READ_ONLY, NO_SYS), - SPEC(home, WSTR_OPT, READ_ONLY, NO_SYS), - SPEC(thread_inherit_context, INT, READ_ONLY, NO_SYS), - SPEC(context_aware_warnings, INT, READ_ONLY, NO_SYS), - SPEC(import_time, UINT, READ_ONLY, NO_SYS), - SPEC(install_signal_handlers, BOOL, READ_ONLY, NO_SYS), - SPEC(isolated, BOOL, READ_ONLY, NO_SYS), // sys.flags.isolated + SPEC(faulthandler, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(filesystem_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(filesystem_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(hash_seed, ULONG, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(home, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(thread_inherit_context, INT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(context_aware_warnings, INT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(import_time, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(install_signal_handlers, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(isolated, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_IsolatedFlag, 0)), // sys.flags.isolated #ifdef MS_WINDOWS - SPEC(legacy_windows_stdio, BOOL, READ_ONLY, NO_SYS), + SPEC(legacy_windows_stdio, BOOL, READ_ONLY, NO_SYS, + GLOBAL(&Py_LegacyWindowsStdioFlag, 0)), #endif - SPEC(malloc_stats, BOOL, READ_ONLY, NO_SYS), - SPEC(pymalloc_hugepages, BOOL, READ_ONLY, NO_SYS), - SPEC(orig_argv, WSTR_LIST, READ_ONLY, SYS_ATTR("orig_argv")), - SPEC(parse_argv, BOOL, READ_ONLY, NO_SYS), - SPEC(pathconfig_warnings, BOOL, READ_ONLY, NO_SYS), - SPEC(perf_profiling, UINT, READ_ONLY, NO_SYS), - SPEC(remote_debug, BOOL, READ_ONLY, NO_SYS), - SPEC(program_name, WSTR, READ_ONLY, NO_SYS), - SPEC(run_command, WSTR_OPT, READ_ONLY, NO_SYS), - SPEC(run_filename, WSTR_OPT, READ_ONLY, NO_SYS), - SPEC(run_module, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(malloc_stats, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(pymalloc_hugepages, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(orig_argv, WSTR_LIST, READ_ONLY, SYS_ATTR("orig_argv"), NO_GLOBAL), + SPEC(parse_argv, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(pathconfig_warnings, BOOL, READ_ONLY, NO_SYS, + GLOBAL(&Py_FrozenFlag, 1)), + SPEC(perf_profiling, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(remote_debug, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(program_name, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(run_command, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(run_filename, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(run_module, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), #ifdef Py_DEBUG - SPEC(run_presite, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(run_presite, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), #endif - SPEC(safe_path, BOOL, READ_ONLY, NO_SYS), - SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS), - SPEC(site_import, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_site - SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS), - SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS), - SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS), - SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS), - SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS), - SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS), + SPEC(safe_path, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(site_import, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_NoSiteFlag, 1)), // sys.flags.no_site + SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), #ifdef __APPLE__ - SPEC(use_system_logger, BOOL, READ_ONLY, NO_SYS), + SPEC(use_system_logger, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), #endif - SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_user_site - SPEC(warn_default_encoding, BOOL, READ_ONLY, NO_SYS), + SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS, + GLOBAL(&Py_NoUserSiteDirectory, 1)), // sys.flags.no_user_site + SPEC(warn_default_encoding, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), // --- Init-only options ----------- - SPEC(_config_init, UINT, INIT_ONLY, NO_SYS), - SPEC(_init_main, BOOL, INIT_ONLY, NO_SYS), - SPEC(_install_importlib, BOOL, INIT_ONLY, NO_SYS), - SPEC(_is_python_build, BOOL, INIT_ONLY, NO_SYS), - SPEC(module_search_paths_set, BOOL, INIT_ONLY, NO_SYS), - SPEC(pythonpath_env, WSTR_OPT, INIT_ONLY, NO_SYS), - SPEC(sys_path_0, WSTR_OPT, INIT_ONLY, NO_SYS), + SPEC(_config_init, UINT, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(_init_main, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(_install_importlib, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(_is_python_build, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(module_search_paths_set, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(pythonpath_env, WSTR_OPT, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(sys_path_0, WSTR_OPT, INIT_ONLY, NO_SYS, NO_GLOBAL), // Array terminator {NULL, 0, 0, 0, NO_SYS}, @@ -233,11 +252,16 @@ static const PyConfigSpec PYPRECONFIG_SPEC[] = { {NULL, 0, 0, 0, NO_SYS}, }; +// End of ignoring deprecations on global variables +_Py_COMP_DIAG_POP + #undef SPEC #undef SYS_ATTR #undef SYS_FLAG_SETTER #undef SYS_FLAG #undef NO_SYS +#undef GLOBAL +#undef NO_GLOBAL // Forward declarations @@ -4951,6 +4975,16 @@ PyConfig_Set(const char *name, PyObject *value) Py_UNREACHABLE(); } + // Set the global variable + if (spec->global_var.ptr != NULL) { + assert(has_int_value); + int value = int_value; + if (spec->global_var.not) { + value = !value; + } + *spec->global_var.ptr = value; + } + if (spec->sys.attr != NULL) { // Set the sys attribute, but don't set PyInterpreterState.config // to keep the code simple.

1 0