From 1b01a914f7c8fef34921cc2e4b4cdc8dc90625dc Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Tue, 24 Sep 2024 15:13:32 -0700 Subject: [PATCH 01/84] Add `f_generator` property to Python frame objects `f_generator` returns the generator / coroutine / async generator object that owns the frame. For all other kinds of frames it will return `None`. This is useful to reconstruct call stack for async/await code. --- Doc/library/inspect.rst | 10 +++++++++ Lib/test/test_frame.py | 50 +++++++++++++++++++++++++++++++++++++++++ Objects/frameobject.c | 11 +++++++++ 3 files changed, 71 insertions(+) diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 853671856b2a144..6ed76faa6ebcdd4 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -162,6 +162,12 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | per-opcode events are | | | | requested | +-----------------+-------------------+---------------------------+ +| | f_generator | returns the generator or | +| | | coroutine object that | +| | | owns this frame, or | +| | | ``None`` if the frame is | +| | | of a regular function | ++-----------------+-------------------+---------------------------+ | | clear() | used to clear all | | | | references to local | | | | variables | @@ -310,6 +316,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Add ``__builtins__`` attribute to functions. +.. versionchanged:: 3.14 + + Add ``f_generator`` attribute to frames. + .. function:: getmembers(object[, predicate]) Return all the members of an object in a list of ``(name, value)`` diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 32de8ed9a13f80a..bec5655180855de 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -222,6 +222,56 @@ def test_f_lineno_del_segfault(self): with self.assertRaises(AttributeError): del f.f_lineno + def test_f_generator(self): + # Test f_generator in different contexts. + + def t0(): + def nested(): + frame = sys._getframe() + return frame.f_generator + + def gen(): + yield nested() + + g = gen() + try: + return next(g) + finally: + g.close() + + def t1(): + frame = sys._getframe() + return frame.f_generator + + def t2(): + frame = sys._getframe() + yield frame.f_generator + + async def t3(): + frame = sys._getframe() + return frame.f_generator + + # For regular functions f_generator is None + self.assertIsNone(t0()) + self.assertIsNone(t1()) + + # For generators f_generator is equal to self + g = t2() + try: + frame_g = next(g) + self.assertIs(g, frame_g) + finally: + g.close() + + # Ditto for coroutines + c = t3() + try: + c.send(None) + except StopIteration as ex: + self.assertIs(ex.value, c) + else: + raise AssertionError('coroutine did not exit') + class ReprTest(unittest.TestCase): """ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index f3a66ffc9aac8f4..7119405d4971f7d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1616,6 +1616,16 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure) return 0; } +static PyObject * +frame_getgenerator(PyFrameObject *f, void *arg) { + if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { + PyObject *gen = (PyObject *)_PyGen_GetGeneratorFromFrame(f->f_frame); + Py_INCREF(gen); + return gen; + } + Py_RETURN_NONE; +} + static PyGetSetDef frame_getsetlist[] = { {"f_back", (getter)frame_getback, NULL, NULL}, @@ -1628,6 +1638,7 @@ static PyGetSetDef frame_getsetlist[] = { {"f_builtins", (getter)frame_getbuiltins, NULL, NULL}, {"f_code", (getter)frame_getcode, NULL, NULL}, {"f_trace_opcodes", (getter)frame_gettrace_opcodes, (setter)frame_settrace_opcodes, NULL}, + {"f_generator", (getter)frame_getgenerator, NULL, NULL}, {0} }; From 0fc5511578b2c145c6d0fd23615d2004b8ba524e Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 25 Sep 2024 14:00:55 -0700 Subject: [PATCH 02/84] Working implementation of `asyncio.capture_call_stack()` --- .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 4 + Lib/asyncio/__init__.py | 2 + Lib/asyncio/futures.py | 2 + Lib/asyncio/stack.py | 120 ++++++++ Lib/asyncio/taskgroups.py | 6 + Lib/asyncio/tasks.py | 37 ++- Lib/test/test_asyncio/test_stack.py | 262 ++++++++++++++++++ Modules/_asynciomodule.c | 232 +++++++++++++++- 11 files changed, 663 insertions(+), 5 deletions(-) create mode 100644 Lib/asyncio/stack.py create mode 100644 Lib/test/test_asyncio/test_stack.py diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 28a76c36801b4bd..98a48bce511be44 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -741,6 +741,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_asyncio_future_blocking)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_awaited_by)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_blksize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_bootstrap)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_check_retval_)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index ac789b06fb8a616..eddea908ed9709d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -230,6 +230,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) STRUCT_FOR_ID(_asyncio_future_blocking) + STRUCT_FOR_ID(_awaited_by) STRUCT_FOR_ID(_blksize) STRUCT_FOR_ID(_bootstrap) STRUCT_FOR_ID(_check_retval_) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 7847a5c63ebf3f1..3f23898566c6d5c 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -739,6 +739,7 @@ extern "C" { INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ INIT_ID(_asyncio_future_blocking), \ + INIT_ID(_awaited_by), \ INIT_ID(_blksize), \ INIT_ID(_bootstrap), \ INIT_ID(_check_retval_), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index a688f70a2ba36ff..87f7c090f57e03c 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -720,6 +720,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_awaited_by); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_blksize); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 03165a425eb7d25..b05c3fdbdf9641c 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -14,6 +14,7 @@ from .protocols import * from .runners import * from .queues import * +from .stack import * from .streams import * from .subprocess import * from .tasks import * @@ -31,6 +32,7 @@ protocols.__all__ + runners.__all__ + queues.__all__ + + stack.__all__ + streams.__all__ + subprocess.__all__ + tasks.__all__ + diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 5f6fa2348726cfb..bc3e65b35c337a9 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -45,6 +45,8 @@ class Future: """ + _awaited_by = None + # Class variables serving as defaults for instance variables. _state = _PENDING _result = None diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py new file mode 100644 index 000000000000000..e25d9e0a35957ad --- /dev/null +++ b/Lib/asyncio/stack.py @@ -0,0 +1,120 @@ +"""Introspection utils for tasks call stacks.""" + +import dataclasses +import sys +import types + +from . import base_futures +from . import futures +from . import tasks + +__all__ = ( + 'capture_call_stack', + 'FrameCallStackEntry', + 'CoroutineCallStackEntry', + 'FutureCallStack', +) + +# Sadly, we can't re-use the traceback's module datastructures as those +# are tailored for error reporting, whereas we need to represent an +# async call stack. +# +# Going with pretty verbose names as we'd like to export them to the +# top level asyncio namespace, and want to avoid future name clashes. + +@dataclasses.dataclass(slots=True) +class FrameCallStackEntry: + frame: types.FrameType + + +@dataclasses.dataclass(slots=True) +class CoroutineCallStackEntry: + coroutine: types.CoroutineType + + +@dataclasses.dataclass(slots=True) +class FutureCallStack: + future: futures.Future + call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] + awaited_by: list[FutureCallStack] + + +def _build_stack_for_future(future: any) -> FutureCallStack: + if not base_futures.isfuture(future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + + try: + get_coro = future.get_coro + except AttributeError: + coro = None + else: + coro = get_coro() + + st: list[CoroutineCallStackEntry] = [] + awaited_by: list[FutureCallStack] = [] + + while coro is not None: + if hasattr(coro, 'cr_await'): + # A native coroutine or duck-type compatible iterator + st.append(CoroutineCallStackEntry(coro)) + coro = coro.cr_await + elif hasattr(coro, 'ag_await'): + # A native async generator or duck-type compatible iterator + st.append(CoroutineCallStackEntry(coro)) + coro = coro.ag_await + else: + break + + if fut_waiters := getattr(future, '_awaited_by', None): + for parent in fut_waiters: + awaited_by.append(_build_stack_for_future(parent)) + + st.reverse() + return FutureCallStack(future, st, awaited_by) + + +def capture_call_stack(*, future: any = None) -> FutureCallStack | None: + """Capture async call stack for the current task or the provided Future.""" + + if future is not None: + if future is not tasks.current_task(): + return _build_stack_for_future(future) + # else: future is the current task, move on. + else: + future = tasks.current_task() + + if future is None: + # This isn't a generic call stack introspection utility. If we + # can't determine the current task and none was provided, we + # just return. + return None + + call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] = [] + + f = sys._getframe(1) + try: + while f is not None: + is_async = f.f_generator is not None + + if is_async: + call_stack.append(CoroutineCallStackEntry(f.f_generator)) + if f.f_back is not None and f.f_back.f_generator is None: + # We've reached the bottom of the coroutine stack, which + # must be the Task that runs it. + break + else: + call_stack.append(FrameCallStackEntry(f)) + + f = f.f_back + finally: + del f + + awaited_by = [] + if getattr(future, '_awaited_by', None): + for parent in future._awaited_by: + awaited_by.append(_build_stack_for_future(parent)) + + return FutureCallStack(future, call_stack, awaited_by) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index f2ee9648c43876d..a5c936ef85f63cf 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -174,6 +174,9 @@ def create_task(self, coro, *, name=None, context=None): else: task = self._loop.create_task(coro, name=name, context=context) + if hasattr(task, '_awaited_by'): + task._awaited_by = {self._parent_task} + # optimization: Immediately call the done callback if the task is # already done (e.g. if the coro was able to complete eagerly), # and skip scheduling a done callback @@ -202,6 +205,9 @@ def _abort(self): def _on_task_done(self, task): self._tasks.discard(task) + if hasattr(task, '_awaited_by'): + task._awaited_by = None + if self._on_completed_fut is not None and not self._tasks: if not self._on_completed_fut.done(): self._on_completed_fut.set_result(True) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 2112dd4b99d17f5..ac2e18adfdabb00 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -322,6 +322,7 @@ def __step_run_and_handle_result(self, exc): self._loop.call_soon( self.__step, new_exc, context=self._context) else: + _add_to_awaited_by(result, self) result._asyncio_future_blocking = False result.add_done_callback( self.__wakeup, context=self._context) @@ -356,6 +357,7 @@ def __step_run_and_handle_result(self, exc): self = None # Needed to break cycles when an exception occurs. def __wakeup(self, future): + _discard_from_awaited_by(future, self) try: future.result() except BaseException as exc: @@ -502,6 +504,7 @@ async def _wait(fs, timeout, return_when, loop): if timeout is not None: timeout_handle = loop.call_later(timeout, _release_waiter, waiter) counter = len(fs) + cur_task = current_task() def _on_completion(f): nonlocal counter @@ -514,9 +517,11 @@ def _on_completion(f): timeout_handle.cancel() if not waiter.done(): waiter.set_result(None) + _discard_from_awaited_by(f, cur_task) for f in fs: f.add_done_callback(_on_completion) + _add_to_awaited_by(f, cur_task) try: await waiter @@ -802,10 +807,13 @@ def gather(*coros_or_futures, return_exceptions=False): outer.set_result([]) return outer - def _done_callback(fut): + def _done_callback(fut, cur_task): nonlocal nfinished nfinished += 1 + if cur_task is not None: + _discard_from_awaited_by(fut, cur_task) + if outer is None or outer.done(): if not fut.cancelled(): # Mark exception retrieved. @@ -864,6 +872,7 @@ def _done_callback(fut): done_futs = [] loop = None outer = None # bpo-46672 + cur_task = current_task() for arg in coros_or_futures: if arg not in arg_to_fut: fut = ensure_future(arg, loop=loop) @@ -875,13 +884,14 @@ def _done_callback(fut): # can't control it, disable the "destroy pending task" # warning. fut._log_destroy_pending = False - + if cur_task is not None: + _add_to_awaited_by(fut, cur_task) nfuts += 1 arg_to_fut[arg] = fut if fut.done(): done_futs.append(fut) else: - fut.add_done_callback(_done_callback) + fut.add_done_callback(lambda fut: _done_callback(fut, cur_task)) else: # There's a duplicate Future object in coros_or_futures. @@ -940,7 +950,14 @@ def shield(arg): loop = futures._get_loop(inner) outer = loop.create_future() - def _inner_done_callback(inner): + cur_task = current_task() + if cur_task is not None: + _add_to_awaited_by(inner, cur_task) + + def _inner_done_callback(inner, cur_task=cur_task): + if cur_task is not None: + _discard_from_awaited_by(inner, cur_task) + if outer.cancelled(): if not inner.cancelled(): # Mark inner's result as retrieved. @@ -1074,6 +1091,18 @@ def _unregister_eager_task(task): _eager_tasks.discard(task) +def _add_to_awaited_by(fut, waiter): + if hasattr(fut, '_awaited_by'): + if fut._awaited_by is None: + fut._awaited_by = set() + fut._awaited_by.add(waiter) + + +def _discard_from_awaited_by(fut, waiter): + if awaited_by := getattr(fut, '_awaited_by', None): + awaited_by.discard(waiter) + + _py_current_task = current_task _py_register_task = _register_task _py_register_eager_task = _register_eager_task diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py new file mode 100644 index 000000000000000..f669772932f84df --- /dev/null +++ b/Lib/test/test_asyncio/test_stack.py @@ -0,0 +1,262 @@ +import asyncio +import unittest + +import pprint + + +# To prevent a warning "test altered the execution environment" +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + +def capture_test_stack(*, fut=None): + + def walk(s): + ret = [ + f"T<{n}>" if '-' not in (n := s.future.get_name()) else 'T' + if isinstance(s.future, asyncio.Task) else 'F' + ] + + ret.append( + [ + ( + f"s {entry.frame.f_code.co_name}" + if isinstance(entry, asyncio.FrameCallStackEntry) else + ( + f"a {entry.coroutine.cr_code.co_name}" + if hasattr(entry.coroutine, 'cr_code') else + f"ag {entry.coroutine.ag_code.co_name}" + ) + ) for entry in s.call_stack + ] + ) + + ret.append( + sorted([ + walk(ab) for ab in s.awaited_by + ], key=lambda entry: entry[0]) + ) + + return ret + + stack = asyncio.capture_call_stack(future=fut) + return walk(stack) + + +class TestCallStack(unittest.IsolatedAsyncioTestCase): + + + async def test_stack_tgroup(self): + + stack_for_c5 = None + + def c5(): + nonlocal stack_for_c5 + stack_for_c5 = capture_test_stack() + + async def c4(): + await asyncio.sleep(0) + c5() + + async def c3(): + await c4() + + async def c2(): + await c3() + + async def c1(task): + await task + + async def main(): + async with asyncio.TaskGroup() as tg: + task = tg.create_task(c2(), name="c2_root") + tg.create_task(c1(task), name="sub_main_1") + tg.create_task(c1(task), name="sub_main_2") + + await main() + + self.assertEqual(stack_for_c5, [ + # task name + 'T', + # call stack + ['s capture_test_stack', 's c5', 'a c4', 'a c3', 'a c2'], + # awaited by + [ + ['T', + ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ], + ['T', + ['a c1'], + [ + ['T', + ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ], + ['T', + ['a c1'], + [ + ['T', + ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ] + ] + ] + ] + ]) + + async def test_stack_async_gen(self): + + stack_for_gen_nested_call = None + + async def gen_nested_call(): + nonlocal stack_for_gen_nested_call + stack_for_gen_nested_call = capture_test_stack() + + async def gen(): + for num in range(2): + yield num + if num == 1: + await gen_nested_call() + + async def main(): + async for el in gen(): + pass + + await main() + + self.assertEqual(stack_for_gen_nested_call, [ + 'T', + [ + 's capture_test_stack', + 'a gen_nested_call', + 'ag gen', + 'a main', + 'a test_stack_async_gen' + ], + [] + ]) + + async def test_stack_gather(self): + + stack_for_deep = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_deep + stack_for_deep = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(0) + + async def main(): + await asyncio.gather(c1(), c2()) + + await main() + + self.assertEqual(stack_for_deep, [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_gather'], []] + ] + ]) + + async def test_stack_shield(self): + + stack_for_shield = None + + async def deep(): + await asyncio.sleep(0) + nonlocal stack_for_shield + stack_for_shield = capture_test_stack() + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_shield, [ + 'T', + ['s capture_test_stack', 'a deep', 'a c1'], + [ + ['T', ['a main', 'a test_stack_shield'], []] + ] + ]) + + async def test_stack_timeout(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def main(): + await asyncio.shield(c1()) + + await main() + + self.assertEqual(stack_for_inner, [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', ['a main', 'a test_stack_timeout'], []] + ] + ]) + + async def test_stack_wait(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + async with asyncio.timeout(1): + await asyncio.sleep(0) + await inner() + + async def c2(): + for i in range(3): + await asyncio.sleep(0) + + async def main(t1, t2): + while True: + _, pending = await asyncio.wait([t1, t2]) + if not pending: + break + + t1 = asyncio.create_task(c1()) + t2 = asyncio.create_task(c2()) + try: + await main(t1, t2) + finally: + await t1 + await t2 + + self.assertEqual(stack_for_inner, [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [ + ['T', + ['a _wait', 'a wait', 'a main', 'a test_stack_wait'], + [] + ] + ] + ]) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 870084100a1b857..d8b822ef5b53a98 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -40,6 +40,7 @@ typedef enum { PyObject *prefix##_source_tb; \ PyObject *prefix##_cancel_msg; \ PyObject *prefix##_cancelled_exc; \ + PyObject *prefix##_awaited_by; \ fut_state prefix##_state; \ /* These bitfields need to be at the end of the struct so that these and bitfields from TaskObj are contiguous. @@ -475,6 +476,7 @@ future_init(FutureObj *fut, PyObject *loop) Py_CLEAR(fut->fut_source_tb); Py_CLEAR(fut->fut_cancel_msg); Py_CLEAR(fut->fut_cancelled_exc); + Py_CLEAR(fut->fut_awaited_by); fut->fut_state = STATE_PENDING; fut->fut_log_tb = 0; @@ -518,6 +520,216 @@ future_init(FutureObj *fut, PyObject *loop) return 0; } +static int +future_awaited_by_add(FutureObj *fut, PyObject *thing) +{ + /* Most futures/task are only awaited by one entity, so we want + to avoid always creating a set for `fut_awaited_by`. + */ + if (fut->fut_awaited_by == NULL) { + Py_INCREF(thing); + fut->fut_awaited_by = thing; + return 0; + } + + if (PySet_Check(fut->fut_awaited_by)) { + return PySet_Add(fut->fut_awaited_by, thing); + } + + PyObject *set = PySet_New(NULL); + if (set == NULL) { + return -1; + } + if (PySet_Add(set, thing)) { + Py_DECREF(set); + return -1; + } + if (PySet_Add(set, fut->fut_awaited_by)) { + Py_DECREF(set); + return -1; + } + Py_SETREF(fut->fut_awaited_by, set); + return 0; +} + +static int +future_awaited_by_discard(FutureObj *fut, PyObject *thing) +{ + /* Following the semantics of 'set.discard()' here in not + raising an error if `thing` isn't in the `awaited_by` "set". + */ + if (fut->fut_awaited_by == NULL) { + return 0; + } + if (fut->fut_awaited_by == thing) { + Py_CLEAR(fut->fut_awaited_by); + return 0; + } + if (PySet_Check(fut->fut_awaited_by)) { + int err = PySet_Discard(fut->fut_awaited_by, thing); + if (err < 0 && PyErr_Occurred()) { + return -1; + } else { + return 0; + } + } + return 0; +} + +static int +awaited_by_add(asyncio_state *state, PyObject *maybe_fut, PyObject *thing) +{ + if (Future_CheckExact(state, maybe_fut) + || Task_CheckExact(state, maybe_fut) + ) { + return future_awaited_by_add((FutureObj *)maybe_fut, thing); + } + + PyObject *awaited_by; + int err = PyObject_GetOptionalAttr( + maybe_fut, &_Py_ID(_awaited_by), &awaited_by); + if (err < 0) { + return err; + } + + if (err == 1) { + if (PySet_Check(awaited_by)) { + if (PySet_Add(awaited_by, thing)) { + Py_DECREF(awaited_by); + return -1; + } else { + Py_DECREF(awaited_by); + return 0; + } + } else if (awaited_by == Py_None) { + Py_DECREF(awaited_by); + goto new_set; + } else { + Py_DECREF(awaited_by); + PyErr_SetString(PyExc_RuntimeError, + "_awaited_by is not a set or None"); + return -1; + } + } + + assert(err == 0); + assert(awaited_by == NULL); + +new_set: + awaited_by = PySet_New(NULL); + if (awaited_by == NULL) { + return -1; + } + if (PySet_Add(awaited_by, thing)) { + Py_DECREF(awaited_by); + return -1; + } + + err = PyObject_SetAttr(maybe_fut, &_Py_ID(_awaited_by), awaited_by); + Py_DECREF(awaited_by); + return err; +} + +static int +awaited_by_discard(asyncio_state *state, PyObject *maybe_fut, PyObject *thing) +{ + if (Future_CheckExact(state, maybe_fut) + || Task_CheckExact(state, maybe_fut) + ) { + return future_awaited_by_discard((FutureObj *)maybe_fut, thing); + } + + PyObject *awaited_by; + int err = PyObject_GetOptionalAttr( + maybe_fut, &_Py_ID(_awaited_by), &awaited_by); + if (err < 0) { + return err; + } + + if (err == 0) { + return 0; + } + + assert(err == 1); + + if (PySet_Check(awaited_by)) { + err = PySet_Discard(awaited_by, thing); + Py_DECREF(awaited_by); + if (err < 0 && PyErr_Occurred()) { + return -1; + } else { + return 0; + } + } else if (awaited_by == Py_None) { + Py_DECREF(awaited_by); + return 0; + } else { + Py_DECREF(awaited_by); + PyErr_SetString(PyExc_RuntimeError, + "_awaited_by is not a set or None"); + return -1; + } +} + +static PyObject * +future_get_awaited_by(FutureObj *fut) +{ + /* Implementation of a Python getter. */ + if (fut->fut_awaited_by == NULL) { + Py_RETURN_NONE; + } + if (PySet_Check(fut->fut_awaited_by)) { + Py_INCREF(fut->fut_awaited_by); + return fut->fut_awaited_by; + } + + /* We don't want to "leak" our optimization that we don't always create + a set to the pure-Python land. Accessing `_awaited_by` from Python + can mean two things: + + (a) asyncio TaskGroup or gather or a similar primitive uses it + to ensure correct call stack. In this case, the TaskGroup + will attempt to mutate the set. + + (b) an async call stack is being rendered and needs to infer + what tasks are awaiting on this task or future. In this case + we don't want to micro-optimize things. + + The bottom line: it's easier to make a set, use it and return it. + */ + + PyObject *set = PySet_New(NULL); + if (set == NULL) { + return NULL; + } + if (PySet_Add(set, fut->fut_awaited_by)) { + Py_DECREF(set); + return NULL; + } + + Py_SETREF(fut->fut_awaited_by, set); + + Py_INCREF(set); + return set; +} + +static int +future_set_awaited_by(FutureObj *fut, PyObject *set) +{ + /* Implementation of a Python setter. */ + if (set == Py_None) { + Py_CLEAR(fut->fut_awaited_by); + return 0; + } + if (!PySet_Check(set)) { + PyErr_SetString(PyExc_ValueError, "_awaited_by expects a set"); + return -1; + } + Py_XSETREF(fut->fut_awaited_by, set); + Py_INCREF(set); + return 0; +} + static PyObject * future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) { @@ -804,6 +1016,7 @@ FutureObj_clear(FutureObj *fut) Py_CLEAR(fut->fut_source_tb); Py_CLEAR(fut->fut_cancel_msg); Py_CLEAR(fut->fut_cancelled_exc); + Py_CLEAR(fut->fut_awaited_by); PyObject_ClearManagedDict((PyObject *)fut); return 0; } @@ -822,6 +1035,7 @@ FutureObj_traverse(FutureObj *fut, visitproc visit, void *arg) Py_VISIT(fut->fut_source_tb); Py_VISIT(fut->fut_cancel_msg); Py_VISIT(fut->fut_cancelled_exc); + Py_VISIT(fut->fut_awaited_by); PyObject_VisitManagedDict((PyObject *)fut, visit, arg); return 0; } @@ -1504,7 +1718,9 @@ static PyMethodDef FutureType_methods[] = { {"_source_traceback", (getter)FutureObj_get_source_traceback, \ NULL, NULL}, \ {"_cancel_message", (getter)FutureObj_get_cancel_message, \ - (setter)FutureObj_set_cancel_message, NULL}, + (setter)FutureObj_set_cancel_message, NULL}, \ + {"_awaited_by", (getter)future_get_awaited_by, \ + (setter)future_set_awaited_by, NULL}, static PyGetSetDef FutureType_getsetlist[] = { FUTURE_COMMON_GETSETLIST @@ -2198,6 +2414,7 @@ TaskObj_traverse(TaskObj *task, visitproc visit, void *arg) Py_VISIT(fut->fut_source_tb); Py_VISIT(fut->fut_cancel_msg); Py_VISIT(fut->fut_cancelled_exc); + Py_VISIT(fut->fut_awaited_by); PyObject_VisitManagedDict((PyObject *)fut, visit, arg); return 0; } @@ -2938,6 +3155,10 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu goto yield_insteadof_yf; } + if (awaited_by_add(state, result, (PyObject *)task)) { + goto fail; + } + fut->fut_blocking = 0; /* result.add_done_callback(task._wakeup) */ @@ -3016,6 +3237,10 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu goto yield_insteadof_yf; } + if (awaited_by_add(state, result, (PyObject *)task)) { + goto fail; + } + /* result._asyncio_future_blocking = False */ if (PyObject_SetAttr( result, &_Py_ID(_asyncio_future_blocking), Py_False) == -1) { @@ -3213,6 +3438,11 @@ task_wakeup(TaskObj *task, PyObject *o) assert(o); asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); + + if (awaited_by_discard(state, o, (PyObject *)task)) { + return NULL; + } + if (Future_CheckExact(state, o) || Task_CheckExact(state, o)) { PyObject *fut_result = NULL; int res = future_get_result(state, (FutureObj*)o, &fut_result); From 1d20a51dc903459a6b7c02be7959d78c07404ef5 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 12:12:12 -0700 Subject: [PATCH 03/84] Address Guido's comments --- Lib/asyncio/futures.py | 50 ++++++ Lib/asyncio/stack.py | 27 ++-- Lib/asyncio/taskgroups.py | 7 +- Lib/asyncio/tasks.py | 31 ++-- Lib/test/test_asyncio/test_stack.py | 2 - Modules/_asynciomodule.c | 234 ++++++++++++---------------- Modules/clinic/_asynciomodule.c.h | 118 +++++++++++++- 7 files changed, 291 insertions(+), 178 deletions(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index bc3e65b35c337a9..f2150c89d60160a 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -2,6 +2,7 @@ __all__ = ( 'Future', 'wrap_future', 'isfuture', + 'future_add_to_awaited_by', 'future_discard_from_awaited_by', ) import concurrent.futures @@ -68,6 +69,9 @@ class Future: # `yield Future()` (incorrect). _asyncio_future_blocking = False + # Used by the capture_call_stack() API. + _asyncio_awaited_by = None + __log_traceback = False def __init__(self, *, loop=None): @@ -419,6 +423,46 @@ def wrap_future(future, *, loop=None): return new_future +def future_add_to_awaited_by(fut, waiter): + """Record that `fut` is awaited on by `waiter`.""" + # For the sake of keeping the implementation minimal and assuming + # that 99.9% of asyncio users use the built-in Futures and Tasks + # (or their subclasses), we only support native Future objects + # and their subclasses. + # + # Longer version: tracking requires storing the caller-callee + # dependency somewhere. One obvious choice is to store that + # information right in the future itself in a dedicated attribute. + # This means that we'd have to require all duck-type compatible + # futures to implement a specific attribute used by asyncio for + # the book keeping. Another solution would be to store that in + # a global dictionary. The downside here is that that would create + # strong references and any scenario where the "add" call isn't + # followed by a "discard" call would lead to a memory leak. + # Using WeakDict would resolve that issue, but would complicate + # the C code (_asynciomodule.c). The bottom line here is that + # it's not clear that all this work would be worth the effort. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._asyncio_awaited_by is None: + fut._asyncio_awaited_by = set() + fut._asyncio_awaited_by.add(waiter) + + +def future_discard_from_awaited_by(fut, waiter): + """Record that `fut` is no longer awaited on by `waiter`.""" + # See the comment in "future_add_to_awaited_by()" body for + # details on implemntation. + # + # Note that there's an accelerated version of this function + # shadowing this implementation later in this file. + if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): + if fut._asyncio_awaited_by is not None: + fut._asyncio_awaited_by.discard(waiter) + + try: import _asyncio except ImportError: @@ -426,3 +470,9 @@ def wrap_future(future, *, loop=None): else: # _CFuture is needed for tests. Future = _CFuture = _asyncio.Future + +try: + from _asyncio import future_add_to_awaited_by, \ + future_discard_from_awaited_by +except ImportError: + pass diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index e25d9e0a35957ad..fe650d6d0a3cd54 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -1,10 +1,9 @@ """Introspection utils for tasks call stacks.""" -import dataclasses import sys import types +import typing -from . import base_futures from . import futures from . import tasks @@ -22,25 +21,23 @@ # Going with pretty verbose names as we'd like to export them to the # top level asyncio namespace, and want to avoid future name clashes. -@dataclasses.dataclass(slots=True) -class FrameCallStackEntry: + +class FrameCallStackEntry(typing.NamedTuple): frame: types.FrameType -@dataclasses.dataclass(slots=True) -class CoroutineCallStackEntry: +class CoroutineCallStackEntry(typing.NamedTuple): coroutine: types.CoroutineType -@dataclasses.dataclass(slots=True) -class FutureCallStack: +class FutureCallStack(typing.NamedTuple): future: futures.Future call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] awaited_by: list[FutureCallStack] def _build_stack_for_future(future: any) -> FutureCallStack: - if not base_futures.isfuture(future): + if not isinstance(future, futures.Future): raise TypeError( f"{future!r} object does not appear to be compatible " f"with asyncio.Future" @@ -68,7 +65,7 @@ def _build_stack_for_future(future: any) -> FutureCallStack: else: break - if fut_waiters := getattr(future, '_awaited_by', None): + if fut_waiters := getattr(future, '_asyncio_awaited_by', None): for parent in fut_waiters: awaited_by.append(_build_stack_for_future(parent)) @@ -92,6 +89,12 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: # just return. return None + if not isinstance(future, futures.Future): + raise TypeError( + f"{future!r} object does not appear to be compatible " + f"with asyncio.Future" + ) + call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] = [] f = sys._getframe(1) @@ -113,8 +116,8 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: del f awaited_by = [] - if getattr(future, '_awaited_by', None): - for parent in future._awaited_by: + if fut_waiters := getattr(future, '_asyncio_awaited_by', None): + for parent in fut_waiters: awaited_by.append(_build_stack_for_future(parent)) return FutureCallStack(future, call_stack, awaited_by) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index a5c936ef85f63cf..dad5d15256a4efb 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -6,6 +6,7 @@ from . import events from . import exceptions +from . import futures from . import tasks @@ -174,8 +175,7 @@ def create_task(self, coro, *, name=None, context=None): else: task = self._loop.create_task(coro, name=name, context=context) - if hasattr(task, '_awaited_by'): - task._awaited_by = {self._parent_task} + futures.future_add_to_awaited_by(task, self._parent_task) # optimization: Immediately call the done callback if the task is # already done (e.g. if the coro was able to complete eagerly), @@ -205,8 +205,7 @@ def _abort(self): def _on_task_done(self, task): self._tasks.discard(task) - if hasattr(task, '_awaited_by'): - task._awaited_by = None + futures.future_discard_from_awaited_by(task, self._parent_task) if self._on_completed_fut is not None and not self._tasks: if not self._on_completed_fut.done(): diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index ac2e18adfdabb00..a90fc23ff0f551b 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -322,7 +322,7 @@ def __step_run_and_handle_result(self, exc): self._loop.call_soon( self.__step, new_exc, context=self._context) else: - _add_to_awaited_by(result, self) + futures.future_add_to_awaited_by(result, self) result._asyncio_future_blocking = False result.add_done_callback( self.__wakeup, context=self._context) @@ -357,7 +357,7 @@ def __step_run_and_handle_result(self, exc): self = None # Needed to break cycles when an exception occurs. def __wakeup(self, future): - _discard_from_awaited_by(future, self) + futures.future_discard_from_awaited_by(future, self) try: future.result() except BaseException as exc: @@ -517,11 +517,11 @@ def _on_completion(f): timeout_handle.cancel() if not waiter.done(): waiter.set_result(None) - _discard_from_awaited_by(f, cur_task) + futures.future_discard_from_awaited_by(f, cur_task) for f in fs: f.add_done_callback(_on_completion) - _add_to_awaited_by(f, cur_task) + futures.future_add_to_awaited_by(f, cur_task) try: await waiter @@ -812,7 +812,7 @@ def _done_callback(fut, cur_task): nfinished += 1 if cur_task is not None: - _discard_from_awaited_by(fut, cur_task) + futures.future_discard_from_awaited_by(fut, cur_task) if outer is None or outer.done(): if not fut.cancelled(): @@ -885,7 +885,7 @@ def _done_callback(fut, cur_task): # warning. fut._log_destroy_pending = False if cur_task is not None: - _add_to_awaited_by(fut, cur_task) + futures.future_add_to_awaited_by(fut, cur_task) nfuts += 1 arg_to_fut[arg] = fut if fut.done(): @@ -950,13 +950,12 @@ def shield(arg): loop = futures._get_loop(inner) outer = loop.create_future() - cur_task = current_task() - if cur_task is not None: - _add_to_awaited_by(inner, cur_task) + if (cur_task := current_task()) is not None: + futures.future_add_to_awaited_by(inner, cur_task) def _inner_done_callback(inner, cur_task=cur_task): if cur_task is not None: - _discard_from_awaited_by(inner, cur_task) + futures.future_discard_from_awaited_by(inner, cur_task) if outer.cancelled(): if not inner.cancelled(): @@ -1091,18 +1090,6 @@ def _unregister_eager_task(task): _eager_tasks.discard(task) -def _add_to_awaited_by(fut, waiter): - if hasattr(fut, '_awaited_by'): - if fut._awaited_by is None: - fut._awaited_by = set() - fut._awaited_by.add(waiter) - - -def _discard_from_awaited_by(fut, waiter): - if awaited_by := getattr(fut, '_awaited_by', None): - awaited_by.discard(waiter) - - _py_current_task = current_task _py_register_task = _register_task _py_register_eager_task = _register_eager_task diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index f669772932f84df..d7ff9e21b9f256d 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -1,8 +1,6 @@ import asyncio import unittest -import pprint - # To prevent a warning "test altered the execution environment" def tearDownModule(): diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index d8b822ef5b53a98..7726f5726084764 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -75,8 +75,19 @@ typedef struct { #define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) #define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) -#define Future_Check(state, obj) PyObject_TypeCheck(obj, state->FutureType) -#define Task_Check(state, obj) PyObject_TypeCheck(obj, state->TaskType) +#define Future_Check(state, obj) \ + (Future_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->FutureType)) + +#define Task_Check(state, obj) \ + (Task_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->TaskType)) + +#define TaskOrFuture_Check(state, obj) \ + (Task_CheckExact(state, obj) \ + || Future_CheckExact(state, obj) \ + || PyObject_TypeCheck(obj, state->FutureType) \ + || PyObject_TypeCheck(obj, state->TaskType)) #ifdef Py_GIL_DISABLED # define ASYNCIO_STATE_LOCK(state) Py_BEGIN_CRITICAL_SECTION_MUT(&state->mutex) @@ -521,19 +532,28 @@ future_init(FutureObj *fut, PyObject *loop) } static int -future_awaited_by_add(FutureObj *fut, PyObject *thing) +future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) { + if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { + // We only want to support native asyncio Futures. + // For further insight see the comment in the Python + // implementation of "future_add_to_awaited_by()". + return 0; + } + + FutureObj *_fut = (FutureObj *)fut; + /* Most futures/task are only awaited by one entity, so we want to avoid always creating a set for `fut_awaited_by`. */ - if (fut->fut_awaited_by == NULL) { + if (_fut->fut_awaited_by == NULL) { Py_INCREF(thing); - fut->fut_awaited_by = thing; + _fut->fut_awaited_by = thing; return 0; } - if (PySet_Check(fut->fut_awaited_by)) { - return PySet_Add(fut->fut_awaited_by, thing); + if (PySet_Check(_fut->fut_awaited_by)) { + return PySet_Add(_fut->fut_awaited_by, thing); } PyObject *set = PySet_New(NULL); @@ -544,29 +564,38 @@ future_awaited_by_add(FutureObj *fut, PyObject *thing) Py_DECREF(set); return -1; } - if (PySet_Add(set, fut->fut_awaited_by)) { + if (PySet_Add(set, _fut->fut_awaited_by)) { Py_DECREF(set); return -1; } - Py_SETREF(fut->fut_awaited_by, set); + Py_SETREF(_fut->fut_awaited_by, set); return 0; } static int -future_awaited_by_discard(FutureObj *fut, PyObject *thing) +future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) { + if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { + // We only want to support native asyncio Futures. + // For further insight see the comment in the Python + // implementation of "future_add_to_awaited_by()". + return 0; + } + + FutureObj *_fut = (FutureObj *)fut; + /* Following the semantics of 'set.discard()' here in not raising an error if `thing` isn't in the `awaited_by` "set". */ - if (fut->fut_awaited_by == NULL) { + if (_fut->fut_awaited_by == NULL) { return 0; } - if (fut->fut_awaited_by == thing) { - Py_CLEAR(fut->fut_awaited_by); + if (_fut->fut_awaited_by == thing) { + Py_CLEAR(_fut->fut_awaited_by); return 0; } - if (PySet_Check(fut->fut_awaited_by)) { - int err = PySet_Discard(fut->fut_awaited_by, thing); + if (PySet_Check(_fut->fut_awaited_by)) { + int err = PySet_Discard(_fut->fut_awaited_by, thing); if (err < 0 && PyErr_Occurred()) { return -1; } else { @@ -576,101 +605,6 @@ future_awaited_by_discard(FutureObj *fut, PyObject *thing) return 0; } -static int -awaited_by_add(asyncio_state *state, PyObject *maybe_fut, PyObject *thing) -{ - if (Future_CheckExact(state, maybe_fut) - || Task_CheckExact(state, maybe_fut) - ) { - return future_awaited_by_add((FutureObj *)maybe_fut, thing); - } - - PyObject *awaited_by; - int err = PyObject_GetOptionalAttr( - maybe_fut, &_Py_ID(_awaited_by), &awaited_by); - if (err < 0) { - return err; - } - - if (err == 1) { - if (PySet_Check(awaited_by)) { - if (PySet_Add(awaited_by, thing)) { - Py_DECREF(awaited_by); - return -1; - } else { - Py_DECREF(awaited_by); - return 0; - } - } else if (awaited_by == Py_None) { - Py_DECREF(awaited_by); - goto new_set; - } else { - Py_DECREF(awaited_by); - PyErr_SetString(PyExc_RuntimeError, - "_awaited_by is not a set or None"); - return -1; - } - } - - assert(err == 0); - assert(awaited_by == NULL); - -new_set: - awaited_by = PySet_New(NULL); - if (awaited_by == NULL) { - return -1; - } - if (PySet_Add(awaited_by, thing)) { - Py_DECREF(awaited_by); - return -1; - } - - err = PyObject_SetAttr(maybe_fut, &_Py_ID(_awaited_by), awaited_by); - Py_DECREF(awaited_by); - return err; -} - -static int -awaited_by_discard(asyncio_state *state, PyObject *maybe_fut, PyObject *thing) -{ - if (Future_CheckExact(state, maybe_fut) - || Task_CheckExact(state, maybe_fut) - ) { - return future_awaited_by_discard((FutureObj *)maybe_fut, thing); - } - - PyObject *awaited_by; - int err = PyObject_GetOptionalAttr( - maybe_fut, &_Py_ID(_awaited_by), &awaited_by); - if (err < 0) { - return err; - } - - if (err == 0) { - return 0; - } - - assert(err == 1); - - if (PySet_Check(awaited_by)) { - err = PySet_Discard(awaited_by, thing); - Py_DECREF(awaited_by); - if (err < 0 && PyErr_Occurred()) { - return -1; - } else { - return 0; - } - } else if (awaited_by == Py_None) { - Py_DECREF(awaited_by); - return 0; - } else { - Py_DECREF(awaited_by); - PyErr_SetString(PyExc_RuntimeError, - "_awaited_by is not a set or None"); - return -1; - } -} - static PyObject * future_get_awaited_by(FutureObj *fut) { @@ -679,26 +613,10 @@ future_get_awaited_by(FutureObj *fut) Py_RETURN_NONE; } if (PySet_Check(fut->fut_awaited_by)) { - Py_INCREF(fut->fut_awaited_by); - return fut->fut_awaited_by; + return PyFrozenSet_New(fut->fut_awaited_by); } - /* We don't want to "leak" our optimization that we don't always create - a set to the pure-Python land. Accessing `_awaited_by` from Python - can mean two things: - - (a) asyncio TaskGroup or gather or a similar primitive uses it - to ensure correct call stack. In this case, the TaskGroup - will attempt to mutate the set. - - (b) an async call stack is being rendered and needs to infer - what tasks are awaiting on this task or future. In this case - we don't want to micro-optimize things. - - The bottom line: it's easier to make a set, use it and return it. - */ - - PyObject *set = PySet_New(NULL); + PyObject *set = PyFrozenSet_New(NULL); if (set == NULL) { return NULL; } @@ -706,10 +624,6 @@ future_get_awaited_by(FutureObj *fut) Py_DECREF(set); return NULL; } - - Py_SETREF(fut->fut_awaited_by, set); - - Py_INCREF(set); return set; } @@ -1719,8 +1633,8 @@ static PyMethodDef FutureType_methods[] = { NULL, NULL}, \ {"_cancel_message", (getter)FutureObj_get_cancel_message, \ (setter)FutureObj_set_cancel_message, NULL}, \ - {"_awaited_by", (getter)future_get_awaited_by, \ - (setter)future_set_awaited_by, NULL}, + {"_asyncio_awaited_by", (getter)future_get_awaited_by, \ + (setter)future_set_awaited_by, NULL}, static PyGetSetDef FutureType_getsetlist[] = { FUTURE_COMMON_GETSETLIST @@ -3155,7 +3069,7 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu goto yield_insteadof_yf; } - if (awaited_by_add(state, result, (PyObject *)task)) { + if (future_awaited_by_add(state, result, (PyObject *)task)) { goto fail; } @@ -3237,7 +3151,7 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu goto yield_insteadof_yf; } - if (awaited_by_add(state, result, (PyObject *)task)) { + if (future_awaited_by_add(state, result, (PyObject *)task)) { goto fail; } @@ -3439,7 +3353,7 @@ task_wakeup(TaskObj *task, PyObject *o) asyncio_state *state = get_asyncio_state_by_def((PyObject *)task); - if (awaited_by_discard(state, o, (PyObject *)task)) { + if (future_awaited_by_discard(state, o, (PyObject *)task)) { return NULL; } @@ -3903,6 +3817,50 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) return tasks; } +/*[clinic input] +_asyncio.future_add_to_awaited_by + + fut: object + waiter: object + +Record that `fut` is awaited on by `waiter`. + +[clinic start generated code]*/ + +static PyObject * +_asyncio_future_add_to_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter) +/*[clinic end generated code: output=0ab9a1a63389e4df input=29259cdbafe9e7bf]*/ +{ + asyncio_state *state = get_asyncio_state(module); + if (future_awaited_by_add(state, fut, waiter)) { + return NULL; + } + Py_RETURN_NONE; +} + +/*[clinic input] +_asyncio.future_discard_from_awaited_by + + fut: object + waiter: object + +Record that `fut` is no longer awaited on by `waiter`. + +[clinic start generated code]*/ + +static PyObject * +_asyncio_future_discard_from_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter) +/*[clinic end generated code: output=a03b0b4323b779de input=5d67a3edc79b6094]*/ +{ + asyncio_state *state = get_asyncio_state(module); + if (future_awaited_by_discard(state, fut, waiter)) { + return NULL; + } + Py_RETURN_NONE; +} + static int module_traverse(PyObject *mod, visitproc visit, void *arg) { @@ -4072,6 +4030,8 @@ static PyMethodDef asyncio_methods[] = { _ASYNCIO__LEAVE_TASK_METHODDEF _ASYNCIO__SWAP_CURRENT_TASK_METHODDEF _ASYNCIO_ALL_TASKS_METHODDEF + _ASYNCIO_FUTURE_ADD_TO_AWAITED_BY_METHODDEF + _ASYNCIO_FUTURE_DISCARD_FROM_AWAITED_BY_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index d619a124ccead51..8a645d636e8e964 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -1547,4 +1547,120 @@ _asyncio_all_tasks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py exit: return return_value; } -/*[clinic end generated code: output=ffe9b71bc65888b3 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_asyncio_future_add_to_awaited_by__doc__, +"future_add_to_awaited_by($module, /, fut, waiter)\n" +"--\n" +"\n" +"Record that `fut` is awaited on by `waiter`."); + +#define _ASYNCIO_FUTURE_ADD_TO_AWAITED_BY_METHODDEF \ + {"future_add_to_awaited_by", _PyCFunction_CAST(_asyncio_future_add_to_awaited_by), METH_FASTCALL|METH_KEYWORDS, _asyncio_future_add_to_awaited_by__doc__}, + +static PyObject * +_asyncio_future_add_to_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter); + +static PyObject * +_asyncio_future_add_to_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(fut), &_Py_ID(waiter), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"fut", "waiter", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "future_add_to_awaited_by", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *fut; + PyObject *waiter; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + fut = args[0]; + waiter = args[1]; + return_value = _asyncio_future_add_to_awaited_by_impl(module, fut, waiter); + +exit: + return return_value; +} + +PyDoc_STRVAR(_asyncio_future_discard_from_awaited_by__doc__, +"future_discard_from_awaited_by($module, /, fut, waiter)\n" +"--\n" +"\n" +"Record that `fut` is no longer awaited on by `waiter`."); + +#define _ASYNCIO_FUTURE_DISCARD_FROM_AWAITED_BY_METHODDEF \ + {"future_discard_from_awaited_by", _PyCFunction_CAST(_asyncio_future_discard_from_awaited_by), METH_FASTCALL|METH_KEYWORDS, _asyncio_future_discard_from_awaited_by__doc__}, + +static PyObject * +_asyncio_future_discard_from_awaited_by_impl(PyObject *module, PyObject *fut, + PyObject *waiter); + +static PyObject * +_asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(fut), &_Py_ID(waiter), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"fut", "waiter", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "future_discard_from_awaited_by", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *fut; + PyObject *waiter; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + fut = args[0]; + waiter = args[1]; + return_value = _asyncio_future_discard_from_awaited_by_impl(module, fut, waiter); + +exit: + return return_value; +} +/*[clinic end generated code: output=a05f7308434b488c input=a9049054013a1b77]*/ From c8be18e6387f52a757843ef959ca873d106d13bb Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 12:17:41 -0700 Subject: [PATCH 04/84] Add a comment for capture_call_stack() --- Lib/asyncio/stack.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index fe650d6d0a3cd54..09dbe1d38de4c84 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -74,7 +74,34 @@ def _build_stack_for_future(future: any) -> FutureCallStack: def capture_call_stack(*, future: any = None) -> FutureCallStack | None: - """Capture async call stack for the current task or the provided Future.""" + """Capture async call stack for the current task or the provided Future. + + The stack is represented with three data structures: + + * FutureCallStack(future, call_stack, awaited_by) + + Where 'future' is a reference to an asyncio.Future or asyncio.Task + (or their subclasses.) + + 'call_stack' is a list of FrameCallStackEntry and CoroutineCallStackEntry + objects (more on them below.) + + 'awaited_by' is a list of FutureCallStack objects. + + * FrameCallStackEntry(frame) + + Where 'frame' is a frame object of a regular Python function + in the call stack. + + * CoroutineCallStackEntry(coroutine) + + Where 'coroutine' is a coroutine object of an awaiting coroutine + or asyncronous generator. + + Receives an optional keyword-only "future" argument. If not passed, + the current task will be used. If there's no current task, the function + returns None. + """ if future is not None: if future is not tasks.current_task(): From abf2cb9b42c5618f7a6b2208bf9c89548b9a95cc Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 12:30:41 -0700 Subject: [PATCH 05/84] Add a couple more tests --- Lib/test/test_asyncio/test_stack.py | 74 ++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index d7ff9e21b9f256d..f64a444f0a7697f 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -11,7 +11,7 @@ def capture_test_stack(*, fut=None): def walk(s): ret = [ - f"T<{n}>" if '-' not in (n := s.future.get_name()) else 'T' + (f"T<{n}>" if '-' not in (n := s.future.get_name()) else 'T') if isinstance(s.future, asyncio.Task) else 'F' ] @@ -258,3 +258,75 @@ async def main(t1, t2): ] ] ]) + + async def test_stack_task(self): + + stack_for_inner = None + + async def inner(): + await asyncio.sleep(0) + nonlocal stack_for_inner + stack_for_inner = capture_test_stack() + + async def c1(): + await inner() + + async def c2(): + await asyncio.create_task(c1(), name='there there') + + async def main(): + await c2() + + await main() + + self.assertEqual(stack_for_inner, [ + 'T', + ['s capture_test_stack', 'a inner', 'a c1'], + [['T', ['a c2', 'a main', 'a test_stack_task'], []]] + ]) + + async def test_stack_future(self): + + stack_for_fut = None + + async def a2(fut): + await fut + + async def a1(fut): + await a2(fut) + + async def b1(fut): + await fut + + async def main(): + nonlocal stack_for_fut + + fut = asyncio.Future() + async with asyncio.TaskGroup() as g: + g.create_task(a1(fut), name="task A") + g.create_task(b1(fut), name='task B') + + for _ in range(5): + # Do a few iterations to ensure that both a1 and b1 + # await on the future + await asyncio.sleep(0) + + stack_for_fut = capture_test_stack(fut=fut) + fut.set_result(None) + + await main() + + self.assertEqual(stack_for_fut, + ['F', + [], + [ + ['T', + ['a a2', 'a a1'], + [['T', ['a test_stack_future'], []]] + ], + ['T', + ['a b1'], + [['T', ['a test_stack_future'], []]] + ], + ]] + ) From 20ceab7ba5a7391c3ef07754d71a84141d7f8d9e Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 12:50:54 -0700 Subject: [PATCH 06/84] Remove setter for C impl of Task._awaited_by --- Modules/_asynciomodule.c | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 7726f5726084764..105c8cb64e6326a 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -627,23 +627,6 @@ future_get_awaited_by(FutureObj *fut) return set; } -static int -future_set_awaited_by(FutureObj *fut, PyObject *set) -{ - /* Implementation of a Python setter. */ - if (set == Py_None) { - Py_CLEAR(fut->fut_awaited_by); - return 0; - } - if (!PySet_Check(set)) { - PyErr_SetString(PyExc_ValueError, "_awaited_by expects a set"); - return -1; - } - Py_XSETREF(fut->fut_awaited_by, set); - Py_INCREF(set); - return 0; -} - static PyObject * future_set_result(asyncio_state *state, FutureObj *fut, PyObject *res) { @@ -1633,8 +1616,7 @@ static PyMethodDef FutureType_methods[] = { NULL, NULL}, \ {"_cancel_message", (getter)FutureObj_get_cancel_message, \ (setter)FutureObj_set_cancel_message, NULL}, \ - {"_asyncio_awaited_by", (getter)future_get_awaited_by, \ - (setter)future_set_awaited_by, NULL}, + {"_asyncio_awaited_by", (getter)future_get_awaited_by, NULL, NULL}, static PyGetSetDef FutureType_getsetlist[] = { FUTURE_COMMON_GETSETLIST From 72d9321d8ad08571fd73ea672f68498af014bb8d Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 15:06:30 -0700 Subject: [PATCH 07/84] Intoduce cr_task --- Include/cpython/genobject.h | 2 ++ Include/internal/pycore_genobject.h | 7 +++++++ Lib/test/test_sys.py | 2 +- Modules/_asynciomodule.c | 28 ++++++++++++++++++++++++---- Objects/genobject.c | 15 ++++++++++++++- 5 files changed, 48 insertions(+), 6 deletions(-) diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index f75884e597e2c24..f0c36081d120cc3 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -32,6 +32,8 @@ PyAPI_DATA(PyTypeObject) PyCoro_Type; PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, PyObject *name, PyObject *qualname); +PyAPI_FUNC(void) _PyCoro_SetTask(PyObject *coro, PyObject *task); + /* --- Asynchronous Generators -------------------------------------------- */ diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index f6d7e6d367177b6..40ef9098124753c 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -22,6 +22,13 @@ extern "C" { PyObject *prefix##_qualname; \ _PyErr_StackItem prefix##_exc_state; \ PyObject *prefix##_origin_or_finalizer; \ + /* A *borrowed* reference to a task that drives the coroutine. \ + The field is meant to be used by profilers and debuggers only. \ + The main invariant is that a task can't get GC'ed while \ + the coroutine it drives is alive and vice versa. \ + Profilers can use this field to reconstruct the full async \ + call stack of program. */ \ + PyObject *prefix##_task; \ char prefix##_hooks_inited; \ char prefix##_closed; \ char prefix##_running_async; \ diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 9689ef8e96e072e..49767964a01977d 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1617,7 +1617,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('6P4c' + INTERPRETER_FRAME + 'P')) + check(get_gen(), size('7P4c' + INTERPRETER_FRAME + 'P')) # iterator check(iter('abc'), size('lP')) # callable-iterator diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 105c8cb64e6326a..553620b1b66cbde 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -239,6 +239,27 @@ static PyObject * task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *result); +static void +clear_task_coro(TaskObj *task) +{ + if (task->task_coro != NULL &&PyCoro_CheckExact(task->task_coro)) { + _PyCoro_SetTask(task->task_coro, (PyObject *)task); + } + Py_CLEAR(task->task_coro); +} + + +static void +set_task_coro(TaskObj *task, PyObject *coro) +{ + if (PyCoro_CheckExact(coro)) { + _PyCoro_SetTask(coro, (PyObject *)task); + } + Py_INCREF(coro); + Py_XSETREF(task->task_coro, coro); +} + + static int _is_coroutine(asyncio_state *state, PyObject *coro) { @@ -2235,8 +2256,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, self->task_must_cancel = 0; self->task_log_destroy_pending = 1; self->task_num_cancels_requested = 0; - Py_INCREF(coro); - Py_XSETREF(self->task_coro, coro); + set_task_coro(self, coro); if (name == Py_None) { // optimization: defer task name formatting @@ -2284,8 +2304,8 @@ static int TaskObj_clear(TaskObj *task) { (void)FutureObj_clear((FutureObj*) task); + clear_task_coro(task); Py_CLEAR(task->task_context); - Py_CLEAR(task->task_coro); Py_CLEAR(task->task_name); Py_CLEAR(task->task_fut_waiter); return 0; @@ -3321,7 +3341,7 @@ task_eager_start(asyncio_state *state, TaskObj *task) register_task(state, task); } else { // This seems to really help performance on pyperformance benchmarks - Py_CLEAR(task->task_coro); + clear_task_coro(task); } return retval; diff --git a/Objects/genobject.c b/Objects/genobject.c index 41cf8fdcc9dee85..bd7090226ec5fa6 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -127,6 +127,10 @@ gen_dealloc(PyGenObject *gen) { PyObject *self = (PyObject *) gen; + /* A borrowed reference used only by coroutines and async + frameworks. Just set it to NULL. */ + gen->gi_task = NULL; + _PyObject_GC_UNTRACK(gen); if (gen->gi_weakreflist != NULL) @@ -961,6 +965,7 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, frame->owner = FRAME_OWNED_BY_GENERATOR; assert(PyObject_GC_IsTracked((PyObject *)f)); Py_DECREF(f); + gen->gi_task = NULL; gen->gi_weakreflist = NULL; gen->gi_exc_state.exc_value = NULL; gen->gi_exc_state.previous_item = NULL; @@ -976,6 +981,13 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, return (PyObject *)gen; } +void +_PyCoro_SetTask(PyObject *coro, PyObject *task) +{ + assert(PyCoro_CheckExact(coro)); + ((PyCoroObject *)coro)->cr_task = task; +} + PyObject * PyGen_NewWithQualName(PyFrameObject *f, PyObject *name, PyObject *qualname) { @@ -1114,7 +1126,6 @@ cr_getcode(PyCoroObject *coro, void *Py_UNUSED(ignored)) return _gen_getcode((PyGenObject *)coro, "cr_code"); } - static PyGetSetDef coro_getsetlist[] = { {"__name__", (getter)gen_get_name, (setter)gen_set_name, PyDoc_STR("name of the coroutine")}, @@ -1348,6 +1359,8 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname) return NULL; } + ((PyCoroObject *)coro)->cr_task = NULL; + PyThreadState *tstate = _PyThreadState_GET(); int origin_depth = tstate->coroutine_origin_tracking_depth; From c9475f6fe8ae047b1e9bd88a7173d00e95718cb9 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 15:27:59 -0700 Subject: [PATCH 08/84] Unbreak shield() and gather() --- Lib/asyncio/tasks.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index a90fc23ff0f551b..4025163416cde36 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -870,9 +870,12 @@ def _done_callback(fut, cur_task): nfuts = 0 nfinished = 0 done_futs = [] - loop = None outer = None # bpo-46672 - cur_task = current_task() + loop = events._get_running_loop() + if loop is not None: + cur_task = current_task(loop) + else: + cur_task = None for arg in coros_or_futures: if arg not in arg_to_fut: fut = ensure_future(arg, loop=loop) @@ -906,7 +909,7 @@ def _done_callback(fut, cur_task): # this will effectively complete the gather eagerly, with the last # callback setting the result (or exception) on outer before returning it for fut in done_futs: - _done_callback(fut) + _done_callback(fut, cur_task) return outer @@ -950,8 +953,10 @@ def shield(arg): loop = futures._get_loop(inner) outer = loop.create_future() - if (cur_task := current_task()) is not None: + if loop is not None and (cur_task := current_task(loop)) is not None: futures.future_add_to_awaited_by(inner, cur_task) + else: + cur_task = None def _inner_done_callback(inner, cur_task=cur_task): if cur_task is not None: From e1099e93538708c505cdc545aba16f7a93d02182 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 26 Sep 2024 15:28:11 -0700 Subject: [PATCH 09/84] Add convinience fields to C Task/Future for profilers --- Modules/_asynciomodule.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 553620b1b66cbde..657ff0f0c2cfe75 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -46,7 +46,11 @@ typedef enum { so that these and bitfields from TaskObj are contiguous. */ \ unsigned prefix##_log_tb: 1; \ - unsigned prefix##_blocking: 1; + unsigned prefix##_blocking: 1; \ + /* Used by profilers to make traversing the stack from an external \ + process faster. */ \ + unsigned prefix##_is_task: 1; \ + unsigned prefix##_awaited_by_is_set: 1; typedef struct { FutureObj_HEAD(fut) @@ -513,6 +517,8 @@ future_init(FutureObj *fut, PyObject *loop) fut->fut_state = STATE_PENDING; fut->fut_log_tb = 0; fut->fut_blocking = 0; + fut->fut_awaited_by_is_set = 0; + fut->fut_is_task = 0; if (loop == Py_None) { asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); @@ -568,12 +574,14 @@ future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) to avoid always creating a set for `fut_awaited_by`. */ if (_fut->fut_awaited_by == NULL) { + assert(!_fut->fut_awaited_by_is_set); Py_INCREF(thing); _fut->fut_awaited_by = thing; return 0; } - if (PySet_Check(_fut->fut_awaited_by)) { + if (_fut->fut_awaited_by_is_set) { + assert(PySet_Check(_fut->fut_awaited_by)); return PySet_Add(_fut->fut_awaited_by, thing); } @@ -590,6 +598,7 @@ future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) return -1; } Py_SETREF(_fut->fut_awaited_by, set); + _fut->fut_awaited_by_is_set = 1; return 0; } @@ -615,7 +624,8 @@ future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) Py_CLEAR(_fut->fut_awaited_by); return 0; } - if (PySet_Check(_fut->fut_awaited_by)) { + if (_fut->fut_awaited_by_is_set) { + assert(PySet_Check(_fut->fut_awaited_by)); int err = PySet_Discard(_fut->fut_awaited_by, thing); if (err < 0 && PyErr_Occurred()) { return -1; @@ -633,7 +643,9 @@ future_get_awaited_by(FutureObj *fut) if (fut->fut_awaited_by == NULL) { Py_RETURN_NONE; } - if (PySet_Check(fut->fut_awaited_by)) { + if (fut->fut_awaited_by_is_set) { + /* Already a set, just wrap it into a frozen set and return. */ + assert(PySet_Check(fut->fut_awaited_by)); return PyFrozenSet_New(fut->fut_awaited_by); } @@ -935,6 +947,7 @@ FutureObj_clear(FutureObj *fut) Py_CLEAR(fut->fut_cancel_msg); Py_CLEAR(fut->fut_cancelled_exc); Py_CLEAR(fut->fut_awaited_by); + fut->fut_awaited_by_is_set = 0; PyObject_ClearManagedDict((PyObject *)fut); return 0; } @@ -2229,6 +2242,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, if (future_init((FutureObj*)self, loop)) { return -1; } + self->task_is_task = 1; asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); int is_coro = is_coroutine(state, coro); From 817f88bba1fdafb79b59adf279ced6cce99c0e26 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 27 Sep 2024 10:06:13 -0700 Subject: [PATCH 10/84] Fix ups --- Modules/_asynciomodule.c | 5 +++-- Objects/genobject.c | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 657ff0f0c2cfe75..7e39841d72d89d7 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -246,8 +246,8 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu static void clear_task_coro(TaskObj *task) { - if (task->task_coro != NULL &&PyCoro_CheckExact(task->task_coro)) { - _PyCoro_SetTask(task->task_coro, (PyObject *)task); + if (task->task_coro != NULL && PyCoro_CheckExact(task->task_coro)) { + _PyCoro_SetTask(task->task_coro, NULL); } Py_CLEAR(task->task_coro); } @@ -256,6 +256,7 @@ clear_task_coro(TaskObj *task) static void set_task_coro(TaskObj *task, PyObject *coro) { + assert(coro != NULL); if (PyCoro_CheckExact(coro)) { _PyCoro_SetTask(coro, (PyObject *)task); } diff --git a/Objects/genobject.c b/Objects/genobject.c index bd7090226ec5fa6..1f2c02884e28aed 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -890,6 +890,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) gen->gi_name = Py_NewRef(func->func_name); assert(func->func_qualname != NULL); gen->gi_qualname = Py_NewRef(func->func_qualname); + gen->gi_task = NULL; _PyObject_GC_TRACK(gen); return (PyObject *)gen; } From 54386ac6a46794d07afaf0da27c62a9e34eac744 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 27 Sep 2024 15:41:11 -0700 Subject: [PATCH 11/84] Add basic docs --- Doc/library/asyncio-future.rst | 2 +- Doc/library/asyncio-stack.rst | 80 +++++++++++++++++++++++++++++++ Doc/library/asyncio.rst | 1 + Lib/asyncio/futures.py | 4 +- Lib/asyncio/stack.py | 12 ++++- Modules/_asynciomodule.c | 6 ++- Modules/clinic/_asynciomodule.c.h | 72 ++++------------------------ 7 files changed, 108 insertions(+), 69 deletions(-) create mode 100644 Doc/library/asyncio-stack.rst diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 9dce07314119401..f1f23b8021b29f8 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -65,7 +65,7 @@ Future Functions and *loop* is not specified and there is no running event loop. -.. function:: wrap_future(future, *, loop=None) +.. function:: wrap_future(future, /, *, loop=None) Wrap a :class:`concurrent.futures.Future` object in a :class:`asyncio.Future` object. diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst new file mode 100644 index 000000000000000..274f64fa2ecfcfa --- /dev/null +++ b/Doc/library/asyncio-stack.rst @@ -0,0 +1,80 @@ +.. currentmodule:: asyncio + + +.. _asyncio-stack: + +=================== +Stack Introspection +=================== + +**Source code:** :source:`Lib/asyncio/stack.py` + +------------------------------------- + +asyncio has powerful runtime call stack introspection utilities +to trace the entire call graph of a running coroutine or task, or +a suspended *future*. + +.. versionadded:: 3.14 + + +.. function:: capture_call_stack(*, future=None) + + Capture the async call stack for the current task or the provided + :class:`Task` or :class:`Future`. + + The function receives an optional keyword-only *future* argument. + If not passed, the current task will be used. If there's no current task, + the function returns ``None``. + + Returns a ``FutureCallStack`` named tuple: + + * ``FutureCallStack(future, call_stack, awaited_by)`` + + Where 'future' is a reference to a *Future* or a *Task* + (or their subclasses.) + + ``call_stack`` is a list of ``FrameCallStackEntry`` and + ``CoroutineCallStackEntry`` objects (more on them below.) + + ``awaited_by`` is a list of ``FutureCallStack`` tuples. + + * ``FrameCallStackEntry(frame)`` + + Where ``frame`` is a frame object of a regular Python function + in the call stack. + + * ``CoroutineCallStackEntry(coroutine)`` + + Where ``coroutine`` is a coroutine object of an awaiting coroutine + or asyncronous generator. + + +Low level utility functions +=========================== + +To introspect an async call stack asyncio requires cooperation from +control flow structures, such as :func:`shield` or :class:`TaskGroup`. +Any time an intermediate ``Future`` object with low-level APIs like +:meth:`Future.add_done_callback() ` is +involved, the following two functions should be used to inform *asyncio* +about how exactly such intermediate future objects are connected with +the tasks they wrap or control. + + +.. function:: future_add_to_awaited_by(future, waiter, /) + + Record that *future* is awaited on by *waiter*. + + Both *future* and *waiter* must be instances of + :class:`asyncio.Future ` or :class:`asyncio.Task ` or + their subclasses, otherwise the call would have no effect. + + +.. function:: future_discard_from_awaited_by(future, waiter, /) + + Record that *future* is no longer awaited on by *waiter*. + + Both *future* and *waiter* must be instances of + :class:`asyncio.Future ` or :class:`asyncio.Task ` or + their subclasses, otherwise the call would have no effect. diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 5f83b3a2658da44..5098805f26cbd51 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -99,6 +99,7 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst + asyncio-stack.rst .. toctree:: :caption: Low-level APIs diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index f2150c89d60160a..5785477248a8d9f 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -423,7 +423,7 @@ def wrap_future(future, *, loop=None): return new_future -def future_add_to_awaited_by(fut, waiter): +def future_add_to_awaited_by(fut, waiter, /): """Record that `fut` is awaited on by `waiter`.""" # For the sake of keeping the implementation minimal and assuming # that 99.9% of asyncio users use the built-in Futures and Tasks @@ -451,7 +451,7 @@ def future_add_to_awaited_by(fut, waiter): fut._asyncio_awaited_by.add(waiter) -def future_discard_from_awaited_by(fut, waiter): +def future_discard_from_awaited_by(fut, waiter, /): """Record that `fut` is no longer awaited on by `waiter`.""" # See the comment in "future_add_to_awaited_by()" body for # details on implemntation. diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 09dbe1d38de4c84..15086df8f63a3a1 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -4,6 +4,7 @@ import types import typing +from . import events from . import futures from . import tasks @@ -103,11 +104,20 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: returns None. """ + loop = events._get_running_loop() + if future is not None: - if future is not tasks.current_task(): + # Check if we're in a context of a running event loop; + # if yes - check if the passed future is the currently + # running task or not. + if loop is None or future is not tasks.current_task(): return _build_stack_for_future(future) # else: future is the current task, move on. else: + if loop is None: + raise RuntimeError( + 'capture_call_stack() is called outside of a running ' + 'event loop and no *future* to introspect was provided') future = tasks.current_task() if future is None: diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 7e39841d72d89d7..0b7d98b0d586b1b 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3839,6 +3839,7 @@ _asyncio.future_add_to_awaited_by fut: object waiter: object + / Record that `fut` is awaited on by `waiter`. @@ -3847,7 +3848,7 @@ Record that `fut` is awaited on by `waiter`. static PyObject * _asyncio_future_add_to_awaited_by_impl(PyObject *module, PyObject *fut, PyObject *waiter) -/*[clinic end generated code: output=0ab9a1a63389e4df input=29259cdbafe9e7bf]*/ +/*[clinic end generated code: output=0ab9a1a63389e4df input=06e6eaac51f532b9]*/ { asyncio_state *state = get_asyncio_state(module); if (future_awaited_by_add(state, fut, waiter)) { @@ -3861,6 +3862,7 @@ _asyncio.future_discard_from_awaited_by fut: object waiter: object + / Record that `fut` is no longer awaited on by `waiter`. @@ -3869,7 +3871,7 @@ Record that `fut` is no longer awaited on by `waiter`. static PyObject * _asyncio_future_discard_from_awaited_by_impl(PyObject *module, PyObject *fut, PyObject *waiter) -/*[clinic end generated code: output=a03b0b4323b779de input=5d67a3edc79b6094]*/ +/*[clinic end generated code: output=a03b0b4323b779de input=b5f7a39ccd36b5db]*/ { asyncio_state *state = get_asyncio_state(module); if (future_awaited_by_discard(state, fut, waiter)) { diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 8a645d636e8e964..f7643f8676cc697 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -1549,53 +1549,26 @@ _asyncio_all_tasks(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py } PyDoc_STRVAR(_asyncio_future_add_to_awaited_by__doc__, -"future_add_to_awaited_by($module, /, fut, waiter)\n" +"future_add_to_awaited_by($module, fut, waiter, /)\n" "--\n" "\n" "Record that `fut` is awaited on by `waiter`."); #define _ASYNCIO_FUTURE_ADD_TO_AWAITED_BY_METHODDEF \ - {"future_add_to_awaited_by", _PyCFunction_CAST(_asyncio_future_add_to_awaited_by), METH_FASTCALL|METH_KEYWORDS, _asyncio_future_add_to_awaited_by__doc__}, + {"future_add_to_awaited_by", _PyCFunction_CAST(_asyncio_future_add_to_awaited_by), METH_FASTCALL, _asyncio_future_add_to_awaited_by__doc__}, static PyObject * _asyncio_future_add_to_awaited_by_impl(PyObject *module, PyObject *fut, PyObject *waiter); static PyObject * -_asyncio_future_add_to_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_future_add_to_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(fut), &_Py_ID(waiter), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"fut", "waiter", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "future_add_to_awaited_by", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; PyObject *fut; PyObject *waiter; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("future_add_to_awaited_by", nargs, 2, 2)) { goto exit; } fut = args[0]; @@ -1607,53 +1580,26 @@ _asyncio_future_add_to_awaited_by(PyObject *module, PyObject *const *args, Py_ss } PyDoc_STRVAR(_asyncio_future_discard_from_awaited_by__doc__, -"future_discard_from_awaited_by($module, /, fut, waiter)\n" +"future_discard_from_awaited_by($module, fut, waiter, /)\n" "--\n" "\n" "Record that `fut` is no longer awaited on by `waiter`."); #define _ASYNCIO_FUTURE_DISCARD_FROM_AWAITED_BY_METHODDEF \ - {"future_discard_from_awaited_by", _PyCFunction_CAST(_asyncio_future_discard_from_awaited_by), METH_FASTCALL|METH_KEYWORDS, _asyncio_future_discard_from_awaited_by__doc__}, + {"future_discard_from_awaited_by", _PyCFunction_CAST(_asyncio_future_discard_from_awaited_by), METH_FASTCALL, _asyncio_future_discard_from_awaited_by__doc__}, static PyObject * _asyncio_future_discard_from_awaited_by_impl(PyObject *module, PyObject *fut, PyObject *waiter); static PyObject * -_asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(fut), &_Py_ID(waiter), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"fut", "waiter", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "future_discard_from_awaited_by", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; PyObject *fut; PyObject *waiter; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("future_discard_from_awaited_by", nargs, 2, 2)) { goto exit; } fut = args[0]; @@ -1663,4 +1609,4 @@ _asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=a05f7308434b488c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e164592826f13567 input=a9049054013a1b77]*/ From 98434f0107d4a81544b70fcb38065f4da2b61cd8 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 27 Sep 2024 17:02:44 -0700 Subject: [PATCH 12/84] Implement, test, and document asyncio.print_call_stack() --- Doc/library/asyncio-stack.rst | 21 ++++++-- Lib/asyncio/stack.py | 78 ++++++++++++++++++++++++++++- Lib/test/test_asyncio/test_stack.py | 33 ++++++++---- 3 files changed, 118 insertions(+), 14 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 274f64fa2ecfcfa..89a3d2b8b218946 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -23,9 +23,9 @@ a suspended *future*. Capture the async call stack for the current task or the provided :class:`Task` or :class:`Future`. - The function receives an optional keyword-only *future* argument. - If not passed, the current task will be used. If there's no current task, - the function returns ``None``. + The function recieves an optional keyword-only *future* argument. + If not passed, the current running task will be used. If there's no + current task, the function returns ``None``. Returns a ``FutureCallStack`` named tuple: @@ -49,6 +49,17 @@ a suspended *future*. Where ``coroutine`` is a coroutine object of an awaiting coroutine or asyncronous generator. +.. function:: print_call_stack(*, future=None, file=None) + + Print the async call stack for the current task or the provided + :class:`Task` or :class:`Future`. + + The function recieves an optional keyword-only *future* argument. + If not passed, the current running task will be used. If there's no + current task, the function returns ``None``. + + If *file* is not specified the function will print to :data:`sys.stdout`. + Low level utility functions =========================== @@ -70,6 +81,10 @@ the tasks they wrap or control. :class:`asyncio.Future ` or :class:`asyncio.Task ` or their subclasses, otherwise the call would have no effect. + A call to ``future_add_to_awaited_by()`` must be followed by an + eventual call to the ``future_discard_from_awaited_by()`` function + with the same arguments. + .. function:: future_discard_from_awaited_by(future, waiter, /) diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 15086df8f63a3a1..16d718e2e9839ba 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -10,6 +10,7 @@ __all__ = ( 'capture_call_stack', + 'print_call_stack', 'FrameCallStackEntry', 'CoroutineCallStackEntry', 'FutureCallStack', @@ -110,7 +111,7 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: # Check if we're in a context of a running event loop; # if yes - check if the passed future is the currently # running task or not. - if loop is None or future is not tasks.current_task(): + if loop is None or future is not tasks.current_task(loop=loop): return _build_stack_for_future(future) # else: future is the current task, move on. else: @@ -118,7 +119,7 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: raise RuntimeError( 'capture_call_stack() is called outside of a running ' 'event loop and no *future* to introspect was provided') - future = tasks.current_task() + future = tasks.current_task(loop=loop) if future is None: # This isn't a generic call stack introspection utility. If we @@ -158,3 +159,76 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: awaited_by.append(_build_stack_for_future(parent)) return FutureCallStack(future, call_stack, awaited_by) + + +def print_call_stack(*, future: any = None, file=None) -> None: + """Print async call stack for the current task or the provided Future.""" + + stack = capture_call_stack(future=future) + if stack is None: + return + + buf = [] + + def render_level(st: FutureCallStack, level: int = 0): + def add_line(line: str): + buf.append(level * ' ' + line) + + if isinstance(st.future, tasks.Task): + add_line( + f'* Task(name={st.future.get_name()!r}, id=0x{id(st.future):x})' + ) + else: + add_line( + f'* Future(id=0x{id(st.future):x})' + ) + + if st.call_stack: + add_line( + f' + Call stack:' + ) + for ste in st.call_stack: + if isinstance(ste, FrameCallStackEntry): + f = ste.frame + add_line( + f' | * {f.f_code.co_qualname}()' + ) + add_line( + f' | {f.f_code.co_filename}:{f.f_lineno}' + ) + else: + assert isinstance(ste, CoroutineCallStackEntry) + c = ste.coroutine + + try: + f = c.cr_frame + code = c.cr_code + tag = 'async' + except AttributeError: + try: + f = c.ag_frame + code = c.ag_code + tag = 'async generator' + except AttributeError: + f = c.gi_frame + code = c.gi_code + tag = 'generator' + + add_line( + f' | * {tag} {code.co_qualname}()' + ) + add_line( + f' | {f.f_code.co_filename}:{f.f_lineno}' + ) + + if st.awaited_by: + add_line( + f' + Awaited by:' + ) + for fut in st.awaited_by: + render_level(fut, level + 1) + + render_level(stack) + rendered = '\n'.join(buf) + + print(rendered, file=file) diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index f64a444f0a7697f..7f434b242f5e442 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -1,4 +1,5 @@ import asyncio +import io import unittest @@ -37,8 +38,11 @@ def walk(s): return ret + buf = io.StringIO() + asyncio.print_call_stack(future=fut, file=buf) + stack = asyncio.capture_call_stack(future=fut) - return walk(stack) + return walk(stack), buf.getvalue() class TestCallStack(unittest.IsolatedAsyncioTestCase): @@ -73,7 +77,7 @@ async def main(): await main() - self.assertEqual(stack_for_c5, [ + self.assertEqual(stack_for_c5[0], [ # task name 'T', # call stack @@ -102,6 +106,11 @@ async def main(): ] ]) + self.assertIn( + '* async TestCallStack.test_stack_tgroup()', + stack_for_c5[1]) + + async def test_stack_async_gen(self): stack_for_gen_nested_call = None @@ -122,7 +131,7 @@ async def main(): await main() - self.assertEqual(stack_for_gen_nested_call, [ + self.assertEqual(stack_for_gen_nested_call[0], [ 'T', [ 's capture_test_stack', @@ -134,6 +143,10 @@ async def main(): [] ]) + self.assertIn( + 'async generator TestCallStack.test_stack_async_gen..gen()', + stack_for_gen_nested_call[1]) + async def test_stack_gather(self): stack_for_deep = None @@ -155,7 +168,7 @@ async def main(): await main() - self.assertEqual(stack_for_deep, [ + self.assertEqual(stack_for_deep[0], [ 'T', ['s capture_test_stack', 'a deep', 'a c1'], [ @@ -181,7 +194,7 @@ async def main(): await main() - self.assertEqual(stack_for_shield, [ + self.assertEqual(stack_for_shield[0], [ 'T', ['s capture_test_stack', 'a deep', 'a c1'], [ @@ -208,7 +221,7 @@ async def main(): await main() - self.assertEqual(stack_for_inner, [ + self.assertEqual(stack_for_inner[0], [ 'T', ['s capture_test_stack', 'a inner', 'a c1'], [ @@ -248,7 +261,7 @@ async def main(t1, t2): await t1 await t2 - self.assertEqual(stack_for_inner, [ + self.assertEqual(stack_for_inner[0], [ 'T', ['s capture_test_stack', 'a inner', 'a c1'], [ @@ -279,7 +292,7 @@ async def main(): await main() - self.assertEqual(stack_for_inner, [ + self.assertEqual(stack_for_inner[0], [ 'T', ['s capture_test_stack', 'a inner', 'a c1'], [['T', ['a c2', 'a main', 'a test_stack_task'], []]] @@ -316,7 +329,7 @@ async def main(): await main() - self.assertEqual(stack_for_fut, + self.assertEqual(stack_for_fut[0], ['F', [], [ @@ -330,3 +343,5 @@ async def main(): ], ]] ) + + self.assertTrue(stack_for_fut[1].startswith('* Future(id=')) From 485c1663dc451c12a2c5e6b301eb86e62c410a31 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sat, 28 Sep 2024 15:20:43 -0700 Subject: [PATCH 13/84] Reorder a few things --- Doc/library/asyncio-stack.rst | 69 +++++++++++++++++++++++++++++------ Lib/asyncio/stack.py | 25 +++++++------ 2 files changed, 72 insertions(+), 22 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 89a3d2b8b218946..c04d80488d35479 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -18,6 +18,64 @@ a suspended *future*. .. versionadded:: 3.14 +.. function:: print_call_stack(*, future=None, file=None) + + Print the async call stack for the current task or the provided + :class:`Task` or :class:`Future`. + + The function recieves an optional keyword-only *future* argument. + If not passed, the current running task will be used. If there's no + current task, the function returns ``None``. + + If *file* is not specified the function will print to :data:`sys.stdout`. + + **Example:** + + The following Python code: + + .. code-block:: python + + import asyncio + + async def test(): + asyncio.print_call_stack() + + async def main(): + async with asyncio.TaskGroup() as g: + g.create_task(test()) + + asyncio.run(main()) + + will print:: + + * Task(name='Task-2', id=0x105038fe0) + + Call stack: + | * print_call_stack() + | asyncio/stack.py:231 + | * async test() + | test.py:4 + + Awaited by: + * Task(name='Task-1', id=0x1050a6060) + + Call stack: + | * async TaskGroup.__aexit__() + | asyncio/taskgroups.py:107 + | * async main() + | test.py:7 + + For rendering the call stack to a string the following pattern + should be used: + + .. code-block:: python + + import io + + ... + + buf = io.StringIO() + asyncio.print_call_stack(file=buf) + output = buf.getvalue() + + .. function:: capture_call_stack(*, future=None) Capture the async call stack for the current task or the provided @@ -49,17 +107,6 @@ a suspended *future*. Where ``coroutine`` is a coroutine object of an awaiting coroutine or asyncronous generator. -.. function:: print_call_stack(*, future=None, file=None) - - Print the async call stack for the current task or the provided - :class:`Task` or :class:`Future`. - - The function recieves an optional keyword-only *future* argument. - If not passed, the current running task will be used. If there's no - current task, the function returns ``None``. - - If *file* is not specified the function will print to :data:`sys.stdout`. - Low level utility functions =========================== diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 16d718e2e9839ba..652e2eb399463c3 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -164,13 +164,7 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: def print_call_stack(*, future: any = None, file=None) -> None: """Print async call stack for the current task or the provided Future.""" - stack = capture_call_stack(future=future) - if stack is None: - return - - buf = [] - - def render_level(st: FutureCallStack, level: int = 0): + def render_level(st: FutureCallStack, buf: list[str], level: int): def add_line(line: str): buf.append(level * ' ' + line) @@ -226,9 +220,18 @@ def add_line(line: str): f' + Awaited by:' ) for fut in st.awaited_by: - render_level(fut, level + 1) + render_level(fut, buf, level + 1) - render_level(stack) - rendered = '\n'.join(buf) + stack = capture_call_stack(future=future) + if stack is None: + return - print(rendered, file=file) + try: + buf = [] + render_level(stack, buf, 0) + rendered = '\n'.join(buf) + print(rendered, file=file) + finally: + # 'stack' has references to frames so we should + # make sure it's GC'ed as soon as we don't need it. + del stack From 8802be7a6db4326568f41adbe1c0db7ff2e0cbac Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 29 Sep 2024 12:23:45 -0700 Subject: [PATCH 14/84] Add a test to exercise asyncio stack traces in out-of-process profilers Signed-off-by: Pablo Galindo --- Include/internal/pycore_runtime.h | 16 + Include/internal/pycore_runtime_init.h | 14 + Lib/test/test_external_inspection.py | 74 ++ Modules/_asynciomodule.c | 49 +- Modules/_testexternalinspection.c | 1117 ++++++++++++++++++++---- 5 files changed, 1112 insertions(+), 158 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index d4291b87261ae0e..c7437a2ad1515c7 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -108,6 +108,7 @@ typedef struct _Py_DebugOffsets { uint64_t instr_ptr; uint64_t localsplus; uint64_t owner; + uint64_t stackpointer; } interpreter_frame; // Code object offset; @@ -152,6 +153,13 @@ typedef struct _Py_DebugOffsets { uint64_t ob_size; } list_object; + // PySet object offset; + struct _set_object { + uint64_t size; + uint64_t used; + uint64_t table; + } set_object; + // PyDict object offset; struct _dict_object { uint64_t size; @@ -192,6 +200,14 @@ typedef struct _Py_DebugOffsets { uint64_t size; uint64_t collecting; } gc; + + struct _gen_object { + uint64_t size; + uint64_t gi_name; + uint64_t gi_iframe; + uint64_t gi_task; + uint64_t gi_frame_state; + } gen_object; } _Py_DebugOffsets; /* Reference tracer state */ diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index e6adb98eb191309..1072196abe4fc91 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -21,6 +21,7 @@ extern "C" { #include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT #include "pycore_signal.h" // _signals_RUNTIME_INIT #include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT +#include "pycore_genobject.h" extern PyTypeObject _PyExc_MemoryError; @@ -73,6 +74,7 @@ extern PyTypeObject _PyExc_MemoryError; .instr_ptr = offsetof(_PyInterpreterFrame, instr_ptr), \ .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ .owner = offsetof(_PyInterpreterFrame, owner), \ + .stackpointer = offsetof(_PyInterpreterFrame, stackpointer), \ }, \ .code_object = { \ .size = sizeof(PyCodeObject), \ @@ -106,6 +108,11 @@ extern PyTypeObject _PyExc_MemoryError; .ob_item = offsetof(PyListObject, ob_item), \ .ob_size = offsetof(PyListObject, ob_base.ob_size), \ }, \ + .set_object = { \ + .size = sizeof(PySetObject), \ + .used = offsetof(PySetObject, used), \ + .table = offsetof(PySetObject, table), \ + }, \ .dict_object = { \ .size = sizeof(PyDictObject), \ .ma_keys = offsetof(PyDictObject, ma_keys), \ @@ -135,6 +142,13 @@ extern PyTypeObject _PyExc_MemoryError; .size = sizeof(struct _gc_runtime_state), \ .collecting = offsetof(struct _gc_runtime_state, collecting), \ }, \ + .gen_object = { \ + .size = sizeof(PyGenObject), \ + .gi_name = offsetof(PyGenObject, gi_name), \ + .gi_iframe = offsetof(PyGenObject, gi_iframe), \ + .gi_task = offsetof(PyGenObject, gi_task), \ + .gi_frame_state = offsetof(PyGenObject, gi_frame_state), \ + }, \ }, \ .allocators = { \ .standard = _pymem_allocators_standard_INIT(runtime), \ diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index d896fec73d19719..d7f55fc6b068415 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -13,6 +13,7 @@ try: from _testexternalinspection import PROCESS_VM_READV_SUPPORTED from _testexternalinspection import get_stack_trace + from _testexternalinspection import get_async_stack_trace except ImportError: raise unittest.SkipTest("Test only runs when _testexternalinspection is available") @@ -74,6 +75,79 @@ def foo(): ] self.assertEqual(stack_trace, expected_stack_trace) + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_async_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import asyncio + import time + import os + import sys + import test.test_asyncio.test_stack as ts + + def c5(): + fifo = sys.argv[1] + with open(sys.argv[1], "w") as fifo: + fifo.write("ready") + time.sleep(10000) + + async def c4(): + await asyncio.sleep(0) + c5() + + async def c3(): + await c4() + + async def c2(): + await c3() + + async def c1(task): + await task + + async def main(): + async with asyncio.TaskGroup() as tg: + task = tg.create_task(c2(), name="c2_root") + tg.create_task(c1(task), name="sub_main_1") + tg.create_task(c1(task), name="sub_main_2") + + asyncio.run(main()) + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + + expected_stack_trace = [ + ["c5", "c4", "c3", "c2"], + "c2_root", + [ + [["main"], "Task-1", []], + [["c1"], "sub_main_2", [[["main"], "Task-1", []]]], + [["c1"], "sub_main_1", [[["main"], "Task-1", []]]], + ], + ] + self.assertEqual(stack_trace, expected_stack_trace) + + + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") def test_self_trace(self): diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 0b7d98b0d586b1b..376664e922fe381 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -16,6 +16,9 @@ #include // offsetof() +#if defined(__APPLE__) +# include +#endif /*[clinic input] module _asyncio @@ -42,15 +45,15 @@ typedef enum { PyObject *prefix##_cancelled_exc; \ PyObject *prefix##_awaited_by; \ fut_state prefix##_state; \ - /* These bitfields need to be at the end of the struct - so that these and bitfields from TaskObj are contiguous. + /* Used by profilers to make traversing the stack from an external \ + process faster. */ \ + char prefix##_is_task; \ + char prefix##_awaited_by_is_set; \ + /* These bitfields need to be at the end of the struct \ + so that these and bitfields from TaskObj are contiguous. \ */ \ unsigned prefix##_log_tb: 1; \ unsigned prefix##_blocking: 1; \ - /* Used by profilers to make traversing the stack from an external \ - process faster. */ \ - unsigned prefix##_is_task: 1; \ - unsigned prefix##_awaited_by_is_set: 1; typedef struct { FutureObj_HEAD(fut) @@ -102,6 +105,40 @@ typedef struct { #endif typedef struct futureiterobject futureiterobject; +typedef struct _Py_AsyncioModuleDebugOffsets { + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; +} Py_AsyncioModuleDebugOffsets; + +#if defined(MS_WINDOWS) + +#pragma section("AsyncioDebug", read, write) +__declspec(allocate("AsyncioDebug")) + +#elif defined(__APPLE__) + +__attribute__((section(SEG_DATA ",AsyncioDebug"))) + +#endif + +Py_AsyncioModuleDebugOffsets AsyncioDebug +#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) + __attribute__((section(".AsyncioDebug"))) +#endif + = {.asyncio_task_object = { + .size = sizeof(TaskObj), + .task_name = offsetof(TaskObj, task_name), + .task_awaited_by = offsetof(TaskObj, task_awaited_by), + .task_is_task = offsetof(TaskObj, task_is_task), + .task_awaited_by_is_set = offsetof(TaskObj, task_awaited_by_is_set), + .task_coro = offsetof(TaskObj, task_coro), + }}; /* State of the _asyncio module */ typedef struct { diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 2476346777c3190..6a14c399933b056 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -57,9 +57,20 @@ # define HAVE_PROCESS_VM_READV 0 #endif +struct _Py_AsyncioModuleDebugOffsets { + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; +}; + #if defined(__APPLE__) && TARGET_OS_OSX -static void* -analyze_macho64(mach_port_t proc_ref, void* base, void* map) +static uintptr_t +return_section_address(const char* section, mach_port_t proc_ref, uintptr_t base, void* map) { struct mach_header_64* hdr = (struct mach_header_64*)map; int ncmds = hdr->ncmds; @@ -72,8 +83,12 @@ analyze_macho64(mach_port_t proc_ref, void* base, void* map) mach_vm_address_t address = (mach_vm_address_t)base; vm_region_basic_info_data_64_t region_info; mach_port_t object_name; + uintptr_t vmaddr = 0; for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { + if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__TEXT") == 0) { + vmaddr = cmd->vmaddr; + } if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { while (cmd->filesize != size) { address += size; @@ -88,17 +103,16 @@ analyze_macho64(mach_port_t proc_ref, void* base, void* map) != KERN_SUCCESS) { PyErr_SetString(PyExc_RuntimeError, "Cannot get any more VM maps.\n"); - return NULL; + return 0; } } - base = (void*)address - cmd->vmaddr; int nsects = cmd->nsects; struct section_64* sec = (struct section_64*)((void*)cmd + sizeof(struct segment_command_64)); for (int j = 0; j < nsects; j++) { - if (strcmp(sec[j].sectname, "PyRuntime") == 0) { - return base + sec[j].addr; + if (strcmp(sec[j].sectname, section) == 0) { + return base + sec[j].addr - vmaddr; } } cmd_cnt++; @@ -106,33 +120,33 @@ analyze_macho64(mach_port_t proc_ref, void* base, void* map) cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize); } - return NULL; + return 0; } -static void* -analyze_macho(char* path, void* base, mach_vm_size_t size, mach_port_t proc_ref) +static uintptr_t +search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_size_t size, mach_port_t proc_ref) { int fd = open(path, O_RDONLY); if (fd == -1) { PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); - return NULL; + return 0; } struct stat fs; if (fstat(fd, &fs) == -1) { PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); close(fd); - return NULL; + return 0; } void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); if (map == MAP_FAILED) { PyErr_Format(PyExc_RuntimeError, "Cannot map binary %s\n", path); close(fd); - return NULL; + return 0; } - void* result = NULL; + uintptr_t result = 0; struct mach_header_64* hdr = (struct mach_header_64*)map; switch (hdr->magic) { @@ -144,7 +158,7 @@ analyze_macho(char* path, void* base, mach_vm_size_t size, mach_port_t proc_ref) break; case MH_MAGIC_64: case MH_CIGAM_64: - result = analyze_macho64(proc_ref, base, map); + result = return_section_address(secname, proc_ref, base, map); break; default: PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic"); @@ -172,9 +186,8 @@ pid_to_task(pid_t pid) return task; } -static void* -get_py_runtime_macos(pid_t pid) -{ +static uintptr_t +search_map_for_section(pid_t pid, const char* secname, const char* substr) { mach_vm_address_t address = 0; mach_vm_size_t size = 0; mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); @@ -184,12 +197,11 @@ get_py_runtime_macos(pid_t pid) mach_port_t proc_ref = pid_to_task(pid); if (proc_ref == 0) { PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID"); - return NULL; + return 0; } int match_found = 0; char map_filename[MAXPATHLEN + 1]; - void* result_address = NULL; while (mach_vm_region( proc_ref, &address, @@ -213,26 +225,21 @@ get_py_runtime_macos(pid_t pid) filename = map_filename; // No path, use the whole string } - // Check if the filename starts with "python" or "libpython" - if (!match_found && strncmp(filename, "python", 6) == 0) { + if (!match_found && strncmp(filename, substr, strlen(substr)) == 0) { match_found = 1; - result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); - } - if (strncmp(filename, "libpython", 9) == 0) { - match_found = 1; - result_address = analyze_macho(map_filename, (void*)address, size, proc_ref); - break; + return search_section_in_file(secname, map_filename, address, size, proc_ref); } address += size; } - return result_address; + return 0; } + #endif #ifdef __linux__ -void* -find_python_map_start_address(pid_t pid, char* result_filename) +static uintptr_t +find_map_start_address(pid_t pid, char* result_filename, const char* map) { char maps_file_path[64]; sprintf(maps_file_path, "/proc/%d/maps", pid); @@ -240,14 +247,14 @@ find_python_map_start_address(pid_t pid, char* result_filename) FILE* maps_file = fopen(maps_file_path, "r"); if (maps_file == NULL) { PyErr_SetFromErrno(PyExc_OSError); - return NULL; + return 0; } int match_found = 0; char line[256]; char map_filename[PATH_MAX]; - void* result_address = 0; + uintptr_t result_address = 0; while (fgets(line, sizeof(line), maps_file) != NULL) { unsigned long start_address = 0; sscanf(line, "%lx-%*x %*s %*s %*s %*s %s", &start_address, map_filename); @@ -258,15 +265,9 @@ find_python_map_start_address(pid_t pid, char* result_filename) filename = map_filename; // No path, use the whole string } - // Check if the filename starts with "python" or "libpython" - if (!match_found && strncmp(filename, "python", 6) == 0) { - match_found = 1; - result_address = (void*)start_address; - strcpy(result_filename, map_filename); - } - if (strncmp(filename, "libpython", 9) == 0) { + if (!match_found && strncmp(filename, map, strlen(map)) == 0) { match_found = 1; - result_address = (void*)start_address; + result_address = start_address; strcpy(result_filename, map_filename); break; } @@ -281,18 +282,17 @@ find_python_map_start_address(pid_t pid, char* result_filename) return result_address; } -void* -get_py_runtime_linux(pid_t pid) +static uintptr_t +search_map_for_section(pid_t pid, const char* secname, const char* map) { char elf_file[256]; - void* start_address = (void*)find_python_map_start_address(pid, elf_file); + uintptr_t start_address = find_map_start_address(pid, elf_file, map); if (start_address == 0) { - PyErr_SetString(PyExc_RuntimeError, "No memory map associated with python or libpython found"); - return NULL; + return 0; } - void* result = NULL; + uintptr_t result = 0; void* file_memory = NULL; int fd = open(elf_file, O_RDONLY); @@ -320,10 +320,13 @@ get_py_runtime_linux(pid_t pid) Elf_Shdr* shstrtab_section = §ion_header_table[elf_header->e_shstrndx]; char* shstrtab = (char*)(file_memory + shstrtab_section->sh_offset); - Elf_Shdr* py_runtime_section = NULL; + Elf_Shdr* section = NULL; for (int i = 0; i < elf_header->e_shnum; i++) { - if (strcmp(".PyRuntime", shstrtab + section_header_table[i].sh_name) == 0) { - py_runtime_section = §ion_header_table[i]; + char* this_sec_name = shstrtab + section_header_table[i].sh_name; + // Move 1 character to account for the leading "." + this_sec_name += 1; + if (strcmp(secname, this_sec_name) == 0) { + section = §ion_header_table[i]; break; } } @@ -338,10 +341,10 @@ get_py_runtime_linux(pid_t pid) } } - if (py_runtime_section != NULL && first_load_segment != NULL) { + if (section != NULL && first_load_segment != NULL) { uintptr_t elf_load_addr = first_load_segment->p_vaddr - (first_load_segment->p_vaddr % first_load_segment->p_align); - result = start_address + py_runtime_section->sh_addr - elf_load_addr; + result = start_address + (uintptr_t)section->sh_addr - elf_load_addr; } exit: @@ -353,10 +356,28 @@ get_py_runtime_linux(pid_t pid) } return result; } + #endif -ssize_t -read_memory(pid_t pid, void* remote_address, size_t len, void* dst) +static uintptr_t +get_py_runtime(pid_t pid) +{ + uintptr_t address = search_map_for_section(pid, "PyRuntime", "libpython"); + if (address == 0) { + address = search_map_for_section(pid, "PyRuntime", "python"); + } + return address; +} + +static uintptr_t +get_async_debug(pid_t pid) +{ + return search_map_for_section(pid, "AsyncioDebug", "_asyncio.cpython"); +} + + +static ssize_t +read_memory(pid_t pid, uintptr_t remote_address, size_t len, void* dst) { ssize_t total_bytes_read = 0; #if defined(__linux__) && HAVE_PROCESS_VM_READV @@ -409,12 +430,16 @@ read_memory(pid_t pid, void* remote_address, size_t len, void* dst) return total_bytes_read; } -int -read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, void* address, char* buffer, Py_ssize_t size) +static int +read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, uintptr_t address, char* buffer, Py_ssize_t size) { Py_ssize_t len; - ssize_t bytes_read = - read_memory(pid, address + debug_offsets->unicode_object.length, sizeof(Py_ssize_t), &len); + ssize_t bytes_read = read_memory( + pid, + address + debug_offsets->unicode_object.length, + sizeof(Py_ssize_t), + &len + ); if (bytes_read == -1) { return -1; } @@ -431,44 +456,604 @@ read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, void* address, char* buf return 0; } -void* -get_py_runtime(pid_t pid) + +static inline int +read_ptr(pid_t pid, uintptr_t address, uintptr_t *ptr_addr) { -#if defined(__linux__) - return get_py_runtime_linux(pid); -#elif defined(__APPLE__) && TARGET_OS_OSX - return get_py_runtime_macos(pid); -#else - return NULL; -#endif + int bytes_read = read_memory(pid, address, sizeof(void*), ptr_addr); + if (bytes_read == -1) { + return -1; + } + return 0; +} + +static inline int +read_ssize_t(pid_t pid, uintptr_t address, Py_ssize_t *size) +{ + int bytes_read = read_memory(pid, address, sizeof(Py_ssize_t), size); + if (bytes_read == -1) { + return -1; + } + return 0; } static int -parse_code_object( - int pid, - PyObject* result, - struct _Py_DebugOffsets* offsets, - void* address, - void** previous_frame) +read_py_ptr(pid_t pid, uintptr_t address, uintptr_t *ptr_addr) +{ + if (read_ptr(pid, address, ptr_addr)) { + return -1; + } + *ptr_addr &= ~Py_TAG_BITS; + return 0; +} + +static int +read_char(pid_t pid, uintptr_t address, char *result) { - void* address_of_function_name; - read_memory( + int bytes_read = read_memory(pid, address, sizeof(char), result); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static int +read_int(pid_t pid, uintptr_t address, int *result) +{ + int bytes_read = read_memory(pid, address, sizeof(int), result); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static int +read_pyobj(pid_t pid, uintptr_t address, PyObject *ptr_addr) +{ + int bytes_read = read_memory(pid, address, sizeof(PyObject), ptr_addr); + if (bytes_read < 0) { + return -1; + } + return 0; +} + +static PyObject * +read_py_str( + pid_t pid, + _Py_DebugOffsets* debug_offsets, + uintptr_t address, + ssize_t max_len +) { + assert(max_len > 0); + + PyObject *result = NULL; + + char *buf = (char *)PyMem_RawMalloc(max_len); + if (buf == NULL) { + PyErr_NoMemory(); + return NULL; + } + if (read_string(pid, debug_offsets, address, buf, max_len)) { + goto err; + } + + result = PyUnicode_FromString(buf); + if (result == NULL) { + goto err; + } + + PyMem_RawFree(buf); + assert(result != NULL); + return result; + +err: + PyMem_RawFree(buf); + return NULL; +} + +static long +read_py_long( + pid_t pid, + _Py_DebugOffsets* offsets, + uintptr_t address) { + unsigned int shift = PYLONG_BITS_IN_DIGIT; + + ssize_t size; + uintptr_t lv_tag; + int bytes_read = read_memory(pid, address + offsets->long_object.lv_tag, sizeof(uintptr_t), &lv_tag); + if (bytes_read == -1) { + return -1; + } + int negative = (lv_tag & 3) == 2; + size = lv_tag >> 3; + + if (size == 0) { + return 0; + } + + char *digits = (char *)PyMem_RawMalloc(size * sizeof(digit)); + if (!digits) { + return -1; + } + bytes_read = read_memory(pid, address + offsets->long_object.ob_digit, sizeof(digit) * size, digits); + if (bytes_read < 0) { + return -1; + } + + long value = 0; + + for (ssize_t i = 0; i < size; ++i) { + long long factor; + if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), &factor)) { + return -1; + } + if (__builtin_add_overflow(value, factor, &value)) { + return -1; + } + } + if (negative) { + value = -1 * value; + } + + return value; +} + +static PyObject * +parse_task_name( + int pid, + _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address +) { + uintptr_t task_name_addr; + int err = read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_name, + &task_name_addr); + if (err) { + return NULL; + } + + // The task name can be a long or a string so we need to check the type + + PyObject task_name_obj; + err = read_pyobj( + pid, + task_name_addr, + &task_name_obj); + if (err) { + return NULL; + } + + int flags; + err = read_int( + pid, + (uintptr_t)task_name_obj.ob_type + offsets->type_object.tp_flags, + &flags); + if (err) { + return NULL; + } + + if ((flags & Py_TPFLAGS_LONG_SUBCLASS)) { + long res = read_py_long(pid, offsets, task_name_addr); + if (res == -1) { + PyErr_SetString(PyExc_RuntimeError, "Failed to get task name"); + return NULL; + } + return PyUnicode_FromFormat("Task-%d", res); + } + + if(!(flags & Py_TPFLAGS_UNICODE_SUBCLASS)) { + PyErr_SetString(PyExc_RuntimeError, "Invalid task name object"); + return NULL; + } + + return read_py_str( + pid, + offsets, + task_name_addr, + 255 + ); +} + +static int +parse_coro_chain( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t coro_address, + PyObject *render_to +) { + assert((void*)coro_address != NULL); + + uintptr_t gen_type_addr; + int err = read_ptr( + pid, + coro_address + sizeof(void*), + &gen_type_addr); + if (err) { + return -1; + } + + uintptr_t gen_name_addr; + err = read_py_ptr( + pid, + coro_address + offsets->gen_object.gi_name, + &gen_name_addr); + if (err) { + return -1; + } + + PyObject *name = read_py_str( + pid, + offsets, + gen_name_addr, + 255 + ); + if (name == NULL) { + return -1; + } + + if (PyList_Append(render_to, name)) { + return -1; + } + + int gi_frame_state; + err = read_int( + pid, + coro_address + offsets->gen_object.gi_frame_state, + &gi_frame_state); + + if (gi_frame_state == FRAME_SUSPENDED_YIELD_FROM) { + char owner; + err = read_char( pid, - (void*)(address + offsets->code_object.name), - sizeof(void*), - &address_of_function_name); + coro_address + offsets->gen_object.gi_iframe + + offsets->interpreter_frame.owner, + &owner + ); + if (err) { + return -1; + } + if (owner != FRAME_OWNED_BY_GENERATOR) { + PyErr_SetString( + PyExc_RuntimeError, + "generator doesn't own its frame \\_o_/"); + return -1; + } - if (address_of_function_name == NULL) { - PyErr_SetString(PyExc_RuntimeError, "No function name found"); + uintptr_t stackpointer_addr; + err = read_py_ptr( + pid, + coro_address + offsets->gen_object.gi_iframe + + offsets->interpreter_frame.stackpointer, + &stackpointer_addr); + if (err) { + return -1; + } + + if ((void*)stackpointer_addr != NULL) { + uintptr_t gi_await_addr; + err = read_py_ptr( + pid, + stackpointer_addr - sizeof(void*), + &gi_await_addr); + if (err) { + return -1; + } + + if ((void*)gi_await_addr != NULL) { + uintptr_t gi_await_addr_type_addr; + int err = read_ptr( + pid, + gi_await_addr + sizeof(void*), + &gi_await_addr_type_addr); + if (err) { + return -1; + } + + if (gen_type_addr == gi_await_addr_type_addr) { + /* This needs an explanation. We always start with parsing + native coroutine / generator frames. Ultimately they + are awaiting on something. That something can be + a native coroutine frame or... an iterator. + If it's the latter -- we can't continue building + our chain. So the condition to bail out of this is + to do that when the type of the current coroutine + doesn't match the type of whatever it points to + in its cr_await. + */ + err = parse_coro_chain( + pid, + offsets, + async_offsets, + gi_await_addr, + render_to + ); + if (err) { + return -1; + } + } + } + } + + } + + return 0; +} + + +static int +parse_task_awaited_by( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address, + PyObject *awaited_by +); + + +static int +parse_task( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address, + PyObject *render_to +) { + char is_task; + int err = read_char( + pid, + task_address + async_offsets->asyncio_task_object.task_is_task, + &is_task); + if (err) { + return -1; + } + + uintptr_t refcnt; + read_ptr(pid, task_address + sizeof(Py_ssize_t), &refcnt); + + PyObject* result = PyList_New(0); + if (result == NULL) { return -1; } - char function_name[256]; - if (read_string(pid, offsets, address_of_function_name, function_name, sizeof(function_name)) != 0) { + PyObject *call_stack = PyList_New(0); + if (call_stack == NULL) { return -1; } + if (PyList_Append(result, call_stack)) { + Py_DECREF(call_stack); + goto err; + } + /* we can operate on a borrowed one to simplify cleanup */ + Py_DECREF(call_stack); + + if (is_task) { + PyObject *tn = parse_task_name(pid, offsets, async_offsets, task_address); + if (tn == NULL) { + goto err; + } + if (PyList_Append(result, tn)) { + Py_DECREF(tn); + goto err; + } + Py_DECREF(tn); + + uintptr_t coro_addr; + err = read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_coro, + &coro_addr); + if (err) { + goto err; + } + + if ((void*)coro_addr != NULL) { + err = parse_coro_chain( + pid, + offsets, + async_offsets, + coro_addr, + call_stack + ); + if (err) { + goto err; + } + + if (PyList_Reverse(call_stack)) { + goto err; + } + } + } - PyObject* py_function_name = PyUnicode_FromString(function_name); + if (PyList_Append(render_to, result)) { + goto err; + } + + PyObject *awaited_by = PyList_New(0); + if (awaited_by == NULL) { + goto err; + } + if (PyList_Append(result, awaited_by)) { + Py_DECREF(awaited_by); + goto err; + } + /* we can operate on a borrowed one to simplify cleanup */ + Py_DECREF(awaited_by); + + if (parse_task_awaited_by(pid, offsets, async_offsets, task_address, awaited_by)) { + goto err; + } + + return 0; + +err: + Py_DECREF(result); + return -1; +} + +static int +parse_tasks_in_set( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t set_addr, + PyObject *awaited_by +) { + uintptr_t set_obj; + if (read_py_ptr( + pid, + set_addr, + &set_obj) + ) { + return -1; + } + + Py_ssize_t set_len; + if (read_ssize_t( + pid, + set_obj + offsets->set_object.used, + &set_len) + ) { + return -1; + } + + Py_ssize_t cnt = 0; + uintptr_t table_ptr; + if (read_ptr( + pid, + set_obj + offsets->set_object.table, + &table_ptr) + ) { + return -1; + } + + while (cnt < set_len) { + uintptr_t key_addr; + if (read_py_ptr(pid, table_ptr, &key_addr)) { + return -1; + } + + if ((void*)key_addr != NULL) { + Py_ssize_t ref_cnt; + if (read_ssize_t(pid, table_ptr, &ref_cnt)) { + return -1; + } + + if (ref_cnt) { + // if 'ref_cnt=0' it's a set dummy marker + + if (parse_task( + pid, + offsets, + async_offsets, + key_addr, + awaited_by) + ) { + return -1; + } + + cnt++; + } + } + + table_ptr += sizeof(void*) * 2; + } + return 0; +} + + +static int +parse_task_awaited_by( + int pid, + struct _Py_DebugOffsets* offsets, + struct _Py_AsyncioModuleDebugOffsets* async_offsets, + uintptr_t task_address, + PyObject *awaited_by +) { + uintptr_t task_ab_addr; + int err = read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_awaited_by, + &task_ab_addr); + if (err) { + return -1; + } + + if ((void*)task_ab_addr == NULL) { + return 0; + } + + char awaited_by_is_a_set; + err = read_char( + pid, + task_address + async_offsets->asyncio_task_object.task_awaited_by_is_set, + &awaited_by_is_a_set); + if (err) { + return -1; + } + + if (awaited_by_is_a_set) { + if (parse_tasks_in_set( + pid, + offsets, + async_offsets, + task_address + async_offsets->asyncio_task_object.task_awaited_by, + awaited_by) + ) { + return -1; + } + } else { + uintptr_t sub_task; + if (read_py_ptr( + pid, + task_address + async_offsets->asyncio_task_object.task_awaited_by, + &sub_task) + ) { + return -1; + } + + if (parse_task( + pid, + offsets, + async_offsets, + sub_task, + awaited_by) + ) { + return -1; + } + } + + return 0; +} + +static int +parse_code_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + uintptr_t address, + uintptr_t* previous_frame +) { + uintptr_t address_of_function_name; + int bytes_read = read_memory( + pid, + address + offsets->code_object.name, + sizeof(void*), + &address_of_function_name + ); + if (bytes_read == -1) { + return -1; + } + + if ((void*)address_of_function_name == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No function name found"); + return -1; + } + + PyObject* py_function_name = read_py_str( + pid, offsets, address_of_function_name, 256); if (py_function_name == NULL) { return -1; } @@ -484,24 +1069,77 @@ parse_code_object( static int parse_frame_object( - int pid, - PyObject* result, - struct _Py_DebugOffsets* offsets, - void* address, - void** previous_frame) -{ + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + uintptr_t address, + uintptr_t* previous_frame +) { + int err; + ssize_t bytes_read = read_memory( - pid, - (void*)(address + offsets->interpreter_frame.previous), - sizeof(void*), - previous_frame); + pid, + address + offsets->interpreter_frame.previous, + sizeof(void*), + previous_frame + ); + if (bytes_read == -1) { + return -1; + } + + char owner; + if (read_char(pid, address + offsets->interpreter_frame.owner, &owner)) { + return -1; + } + + if (owner == FRAME_OWNED_BY_CSTACK) { + return 0; + } + + uintptr_t address_of_code_object; + err = read_py_ptr( + pid, + address + offsets->interpreter_frame.executable, + &address_of_code_object + ); + if (err) { + return -1; + } + + if ((void*)address_of_code_object == NULL) { + return 0; + } + + return parse_code_object( + pid, result, offsets, address_of_code_object, previous_frame); +} + +static int +parse_async_frame_object( + int pid, + PyObject* result, + struct _Py_DebugOffsets* offsets, + uintptr_t address, + uintptr_t* task, + uintptr_t* previous_frame +) { + int err; + + *task = (uintptr_t)NULL; + + ssize_t bytes_read = read_memory( + pid, + address + offsets->interpreter_frame.previous, + sizeof(void*), + previous_frame + ); if (bytes_read == -1) { return -1; } char owner; - bytes_read = - read_memory(pid, (void*)(address + offsets->interpreter_frame.owner), sizeof(char), &owner); + bytes_read = read_memory( + pid, address + offsets->interpreter_frame.owner, sizeof(char), &owner); if (bytes_read < 0) { return -1; } @@ -510,21 +1148,125 @@ parse_frame_object( return 0; } + if (owner == FRAME_OWNED_BY_GENERATOR) { + err = read_py_ptr( + pid, + address - offsets->gen_object.gi_iframe + offsets->gen_object.gi_task, + task); + if (err) { + return -1; + } + } + uintptr_t address_of_code_object; + err = read_py_ptr( + pid, + address + offsets->interpreter_frame.executable, + &address_of_code_object + ); + if (err) { + return -1; + } + + if ((void*)address_of_code_object == NULL) { + return 0; + } + + return parse_code_object( + pid, result, offsets, address_of_code_object, previous_frame); +} + +static int +read_offsets( + int pid, + uintptr_t *runtime_start_address, + _Py_DebugOffsets* debug_offsets +) { + *runtime_start_address = get_py_runtime(pid); + if (!*runtime_start_address) { + if (!PyErr_Occurred()) { + PyErr_SetString( + PyExc_RuntimeError, "Failed to get .PyRuntime address"); + } + return -1; + } + size_t size = sizeof(struct _Py_DebugOffsets); + ssize_t bytes_read = read_memory( + pid, *runtime_start_address, size, debug_offsets); + if (bytes_read == -1) { + return -1; + } + return 0; +} + +static int +read_async_debug( + int pid, + struct _Py_AsyncioModuleDebugOffsets* async_debug +) { + uintptr_t async_debug_addr = get_async_debug(pid); + if (!async_debug) { + return -1; + } + size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); + ssize_t bytes_read = read_memory( + pid, async_debug_addr, size, async_debug); + if (bytes_read == -1) { + return -1; + } + return 0; +} + +static int +find_running_frame( + int pid, + uintptr_t runtime_start_address, + _Py_DebugOffsets* local_debug_offsets, + uintptr_t *frame +) { + off_t interpreter_state_list_head = + local_debug_offsets->runtime_state.interpreters_head; + + uintptr_t address_of_interpreter_state; + int bytes_read = read_memory( + pid, + runtime_start_address + interpreter_state_list_head, + sizeof(void*), + &address_of_interpreter_state); + if (bytes_read == -1) { + return -1; + } + + if (address_of_interpreter_state == 0) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + return -1; + } + + uintptr_t address_of_thread; bytes_read = read_memory( pid, - (void*)(address + offsets->interpreter_frame.executable), + address_of_interpreter_state + + local_debug_offsets->interpreter_state.threads_head, sizeof(void*), - &address_of_code_object); + &address_of_thread); if (bytes_read == -1) { return -1; } - if (address_of_code_object == 0) { + // No Python frames are available for us (can happen at tear-down). + if ((void*)address_of_thread != NULL) { + int err = read_ptr( + pid, + address_of_thread + local_debug_offsets->thread_state.current_frame, + frame); + if (err) { + return -1; + } return 0; } - address_of_code_object &= ~Py_TAG_BITS; - return parse_code_object(pid, result, offsets, (void *)address_of_code_object, previous_frame); + + *frame = (uintptr_t)NULL; + return 0; } static PyObject* @@ -540,88 +1282,159 @@ get_stack_trace(PyObject* self, PyObject* args) return NULL; } - void* runtime_start_address = get_py_runtime(pid); - if (runtime_start_address == NULL) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_RuntimeError, "Failed to get .PyRuntime address"); - } + uintptr_t runtime_start_address = get_py_runtime(pid); + struct _Py_DebugOffsets local_debug_offsets; + + if (read_offsets(pid, &runtime_start_address, &local_debug_offsets)) { return NULL; } - size_t size = sizeof(struct _Py_DebugOffsets); - struct _Py_DebugOffsets local_debug_offsets; - ssize_t bytes_read = read_memory(pid, runtime_start_address, size, &local_debug_offsets); - if (bytes_read == -1) { + uintptr_t address_of_current_frame; + if (find_running_frame( + pid, runtime_start_address, &local_debug_offsets, + &address_of_current_frame) + ) { return NULL; } - off_t interpreter_state_list_head = local_debug_offsets.runtime_state.interpreters_head; - void* address_of_interpreter_state; - bytes_read = read_memory( - pid, - (void*)(runtime_start_address + interpreter_state_list_head), - sizeof(void*), - &address_of_interpreter_state); - if (bytes_read == -1) { + PyObject* result = PyList_New(0); + if (result == NULL) { return NULL; } - if (address_of_interpreter_state == NULL) { - PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + while ((void*)address_of_current_frame != NULL) { + if (parse_frame_object( + pid, + result, + &local_debug_offsets, + address_of_current_frame, + &address_of_current_frame) + < 0) + { + Py_DECREF(result); + return NULL; + } + } + + return result; +} + +static PyObject* +get_async_stack_trace(PyObject* self, PyObject* args) +{ +#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); + return NULL; +#endif + int pid; + + if (!PyArg_ParseTuple(args, "i", &pid)) { return NULL; } - void* address_of_thread; - bytes_read = read_memory( - pid, - (void*)(address_of_interpreter_state + local_debug_offsets.interpreter_state.threads_head), - sizeof(void*), - &address_of_thread); - if (bytes_read == -1) { + uintptr_t runtime_start_address = get_py_runtime(pid); + struct _Py_DebugOffsets local_debug_offsets; + + if (read_offsets(pid, &runtime_start_address, &local_debug_offsets)) { return NULL; } - PyObject* result = PyList_New(0); + struct _Py_AsyncioModuleDebugOffsets local_async_debug; + if (read_async_debug(pid, &local_async_debug)) { + return NULL; + } + + uintptr_t address_of_current_frame; + if (find_running_frame( + pid, runtime_start_address, &local_debug_offsets, + &address_of_current_frame) + ) { + return NULL; + } + + PyObject* result = PyList_New(1); if (result == NULL) { return NULL; } + PyObject* calls = PyList_New(0); + if (calls == NULL) { + return NULL; + } + if (PyList_SetItem(result, 0, calls)) { /* steals ref to 'calls' */ + Py_DECREF(result); + Py_DECREF(calls); + return NULL; + } - // No Python frames are available for us (can happen at tear-down). - if (address_of_thread != NULL) { - void* address_of_current_frame; - (void)read_memory( - pid, - (void*)(address_of_thread + local_debug_offsets.thread_state.current_frame), - sizeof(void*), - &address_of_current_frame); - while (address_of_current_frame != NULL) { - if (parse_frame_object( - pid, - result, - &local_debug_offsets, - address_of_current_frame, - &address_of_current_frame) - < 0) - { - Py_DECREF(result); - return NULL; - } + uintptr_t root_task_addr = (uintptr_t)NULL; + while ((void*)address_of_current_frame != NULL) { + int err = parse_async_frame_object( + pid, + calls, + &local_debug_offsets, + address_of_current_frame, + &root_task_addr, + &address_of_current_frame + ); + if (err) { + goto result_err; + } + + if ((void*)root_task_addr != NULL) { + break; } } + if ((void*)root_task_addr != NULL) { + PyObject *tn = parse_task_name( + pid, &local_debug_offsets, &local_async_debug, root_task_addr); + if (tn == NULL) { + goto result_err; + } + if (PyList_Append(result, tn)) { + Py_DECREF(tn); + goto result_err; + } + Py_DECREF(tn); + + PyObject* awaited_by = PyList_New(0); + if (awaited_by == NULL) { + goto result_err; + } + if (PyList_Append(result, awaited_by)) { + Py_DECREF(awaited_by); + goto result_err; + } + + if (parse_task_awaited_by( + pid, &local_debug_offsets, &local_async_debug, root_task_addr, awaited_by) + ) { + goto result_err; + } + } + + return result; + +result_err: + Py_DECREF(result); + return NULL; } + static PyMethodDef methods[] = { - {"get_stack_trace", get_stack_trace, METH_VARARGS, "Get the Python stack from a given PID"}, - {NULL, NULL, 0, NULL}, + {"get_stack_trace", get_stack_trace, METH_VARARGS, + "Get the Python stack from a given PID"}, + {"get_async_stack_trace", get_async_stack_trace, METH_VARARGS, + "Get the asyncio stack from a given PID"}, + {NULL, NULL, 0, NULL}, }; static struct PyModuleDef module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "_testexternalinspection", - .m_size = -1, - .m_methods = methods, + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_testexternalinspection", + .m_size = -1, + .m_methods = methods, }; PyMODINIT_FUNC From 1ddc9cfad5006f1396d387ecfa7f40b79a8b3fe4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 30 Sep 2024 12:26:38 +0100 Subject: [PATCH 15/84] Refactor the section-generated code into an evil macro Signed-off-by: Pablo Galindo --- Include/internal/pycore_runtime.h | 35 +++++++++++++++++++++++++++++++ Modules/_asynciomodule.c | 20 +----------------- Python/pylifecycle.c | 22 +------------------ 3 files changed, 37 insertions(+), 40 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index c7437a2ad1515c7..b51f427dfa15795 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -25,6 +25,10 @@ extern "C" { #include "pycore_typeobject.h" // struct _types_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state +#if defined(__APPLE__) +# include +#endif + struct _getargs_runtime_state { struct _PyArg_Parser *static_parsers; }; @@ -59,6 +63,37 @@ typedef struct _Py_AuditHookEntry { void *userData; } _Py_AuditHookEntry; +// Macros to burn global values in custom sections so out-of-process +// profilers can locate them easily + +#define GENERATE_DEBUG_SECTION(name, declaration) \ + _GENERATE_DEBUG_SECTION_WINDOWS(name) \ + _GENERATE_DEBUG_SECTION_APPLE(name) \ + declaration \ + _GENERATE_DEBUG_SECTION_LINUX(name) + +#if defined(MS_WINDOWS) +#define _GENERATE_DEBUG_SECTION_WINDOWS(name) \ + _Pragma(Py_STRINGIFY(section(Py_STRINGIFY(name), read, write))) \ + __declspec(allocate(Py_STRINGIFY(name))) +#else +#define _GENERATE_DEBUG_SECTION_WINDOWS(name) +#endif + +#if defined(__APPLE__) +#define _GENERATE_DEBUG_SECTION_APPLE(name) \ + __attribute__((section(SEG_DATA "," Py_STRINGIFY(name)))) +#else +#define _GENERATE_DEBUG_SECTION_APPLE(name) +#endif + +#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) +#define _GENERATE_DEBUG_SECTION_LINUX(name) \ + __attribute__((section("." Py_STRINGIFY(name)))) +#else +#define _GENERATE_DEBUG_SECTION_LINUX(name) +#endif + typedef struct _Py_DebugOffsets { char cookie[8]; uint64_t version; diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 376664e922fe381..032a7a50cf9c797 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -16,10 +16,6 @@ #include // offsetof() -#if defined(__APPLE__) -# include -#endif - /*[clinic input] module _asyncio [clinic start generated code]*/ @@ -116,21 +112,7 @@ typedef struct _Py_AsyncioModuleDebugOffsets { } asyncio_task_object; } Py_AsyncioModuleDebugOffsets; -#if defined(MS_WINDOWS) - -#pragma section("AsyncioDebug", read, write) -__declspec(allocate("AsyncioDebug")) - -#elif defined(__APPLE__) - -__attribute__((section(SEG_DATA ",AsyncioDebug"))) - -#endif - -Py_AsyncioModuleDebugOffsets AsyncioDebug -#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) - __attribute__((section(".AsyncioDebug"))) -#endif +GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets AsyncioDebug) = {.asyncio_task_object = { .size = sizeof(TaskObj), .task_name = offsetof(TaskObj, task_name), diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ebeee4f41d795d2..1c40c9fb2767c66 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -43,10 +43,6 @@ # include // isatty() #endif -#if defined(__APPLE__) -# include -#endif - #ifdef HAVE_SIGNAL_H # include // SIG_IGN #endif @@ -87,23 +83,7 @@ static void call_ll_exitfuncs(_PyRuntimeState *runtime); _Py_COMP_DIAG_PUSH _Py_COMP_DIAG_IGNORE_DEPR_DECLS -#if defined(MS_WINDOWS) - -#pragma section("PyRuntime", read, write) -__declspec(allocate("PyRuntime")) - -#elif defined(__APPLE__) - -__attribute__(( - section(SEG_DATA ",PyRuntime") -)) - -#endif - -_PyRuntimeState _PyRuntime -#if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) -__attribute__ ((section (".PyRuntime"))) -#endif +GENERATE_DEBUG_SECTION(PyRuntime, _PyRuntimeState _PyRuntime) = _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); _Py_COMP_DIAG_POP From bc9beb85aaa7c0631227a5dede062f6f5fdbfe12 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 11:15:55 -0700 Subject: [PATCH 16/84] Rename "capture_call_stack()" et al to "capture_call_graph()" --- Doc/library/asyncio-stack.rst | 30 ++++++------- Lib/asyncio/stack.py | 68 ++++++++++++++--------------- Lib/test/test_asyncio/test_stack.py | 8 ++-- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index c04d80488d35479..675422354e5e92c 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -18,9 +18,9 @@ a suspended *future*. .. versionadded:: 3.14 -.. function:: print_call_stack(*, future=None, file=None) +.. function:: print_call_graph(*, future=None, file=None) - Print the async call stack for the current task or the provided + Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. The function recieves an optional keyword-only *future* argument. @@ -38,7 +38,7 @@ a suspended *future*. import asyncio async def test(): - asyncio.print_call_stack() + asyncio.print_call_graph() async def main(): async with asyncio.TaskGroup() as g: @@ -50,7 +50,7 @@ a suspended *future*. * Task(name='Task-2', id=0x105038fe0) + Call stack: - | * print_call_stack() + | * print_call_graph() | asyncio/stack.py:231 | * async test() | test.py:4 @@ -72,37 +72,37 @@ a suspended *future*. ... buf = io.StringIO() - asyncio.print_call_stack(file=buf) + asyncio.print_call_graph(file=buf) output = buf.getvalue() -.. function:: capture_call_stack(*, future=None) +.. function:: capture_call_graph(*, future=None) - Capture the async call stack for the current task or the provided + Capture the async call graph for the current task or the provided :class:`Task` or :class:`Future`. The function recieves an optional keyword-only *future* argument. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. - Returns a ``FutureCallStack`` named tuple: + Returns a ``FutureCallGraph`` named tuple: - * ``FutureCallStack(future, call_stack, awaited_by)`` + * ``FutureCallGraph(future, call_graph, awaited_by)`` Where 'future' is a reference to a *Future* or a *Task* (or their subclasses.) - ``call_stack`` is a list of ``FrameCallStackEntry`` and - ``CoroutineCallStackEntry`` objects (more on them below.) + ``call_graph`` is a list of ``FrameCallGraphEntry`` and + ``CoroutineCallGraphEntry`` objects (more on them below.) - ``awaited_by`` is a list of ``FutureCallStack`` tuples. + ``awaited_by`` is a list of ``FutureCallGraph`` tuples. - * ``FrameCallStackEntry(frame)`` + * ``FrameCallGraphEntry(frame)`` Where ``frame`` is a frame object of a regular Python function in the call stack. - * ``CoroutineCallStackEntry(coroutine)`` + * ``CoroutineCallGraphEntry(coroutine)`` Where ``coroutine`` is a coroutine object of an awaiting coroutine or asyncronous generator. @@ -111,7 +111,7 @@ a suspended *future*. Low level utility functions =========================== -To introspect an async call stack asyncio requires cooperation from +To introspect an async call graph asyncio requires cooperation from control flow structures, such as :func:`shield` or :class:`TaskGroup`. Any time an intermediate ``Future`` object with low-level APIs like :meth:`Future.add_done_callback() ` is diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 652e2eb399463c3..150638365360ff9 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -9,11 +9,11 @@ from . import tasks __all__ = ( - 'capture_call_stack', - 'print_call_stack', - 'FrameCallStackEntry', - 'CoroutineCallStackEntry', - 'FutureCallStack', + 'capture_call_graph', + 'print_call_graph', + 'FrameCallGraphEntry', + 'CoroutineCallGraphEntry', + 'FutureCallGraph', ) # Sadly, we can't re-use the traceback's module datastructures as those @@ -24,21 +24,21 @@ # top level asyncio namespace, and want to avoid future name clashes. -class FrameCallStackEntry(typing.NamedTuple): +class FrameCallGraphEntry(typing.NamedTuple): frame: types.FrameType -class CoroutineCallStackEntry(typing.NamedTuple): +class CoroutineCallGraphEntry(typing.NamedTuple): coroutine: types.CoroutineType -class FutureCallStack(typing.NamedTuple): +class FutureCallGraph(typing.NamedTuple): future: futures.Future - call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] - awaited_by: list[FutureCallStack] + call_graph: list[FrameCallGraphEntry | CoroutineCallGraphEntry] + awaited_by: list[FutureCallGraph] -def _build_stack_for_future(future: any) -> FutureCallStack: +def _build_stack_for_future(future: any) -> FutureCallGraph: if not isinstance(future, futures.Future): raise TypeError( f"{future!r} object does not appear to be compatible " @@ -52,17 +52,17 @@ def _build_stack_for_future(future: any) -> FutureCallStack: else: coro = get_coro() - st: list[CoroutineCallStackEntry] = [] - awaited_by: list[FutureCallStack] = [] + st: list[CoroutineCallGraphEntry] = [] + awaited_by: list[FutureCallGraph] = [] while coro is not None: if hasattr(coro, 'cr_await'): # A native coroutine or duck-type compatible iterator - st.append(CoroutineCallStackEntry(coro)) + st.append(CoroutineCallGraphEntry(coro)) coro = coro.cr_await elif hasattr(coro, 'ag_await'): # A native async generator or duck-type compatible iterator - st.append(CoroutineCallStackEntry(coro)) + st.append(CoroutineCallGraphEntry(coro)) coro = coro.ag_await else: break @@ -72,30 +72,30 @@ def _build_stack_for_future(future: any) -> FutureCallStack: awaited_by.append(_build_stack_for_future(parent)) st.reverse() - return FutureCallStack(future, st, awaited_by) + return FutureCallGraph(future, st, awaited_by) -def capture_call_stack(*, future: any = None) -> FutureCallStack | None: +def capture_call_graph(*, future: any = None) -> FutureCallGraph | None: """Capture async call stack for the current task or the provided Future. The stack is represented with three data structures: - * FutureCallStack(future, call_stack, awaited_by) + * FutureCallGraph(future, call_graph, awaited_by) Where 'future' is a reference to an asyncio.Future or asyncio.Task (or their subclasses.) - 'call_stack' is a list of FrameCallStackEntry and CoroutineCallStackEntry + 'call_graph' is a list of FrameCallGraphEntry and CoroutineCallGraphEntry objects (more on them below.) - 'awaited_by' is a list of FutureCallStack objects. + 'awaited_by' is a list of FutureCallGraph objects. - * FrameCallStackEntry(frame) + * FrameCallGraphEntry(frame) Where 'frame' is a frame object of a regular Python function in the call stack. - * CoroutineCallStackEntry(coroutine) + * CoroutineCallGraphEntry(coroutine) Where 'coroutine' is a coroutine object of an awaiting coroutine or asyncronous generator. @@ -117,7 +117,7 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: else: if loop is None: raise RuntimeError( - 'capture_call_stack() is called outside of a running ' + 'capture_call_graph() is called outside of a running ' 'event loop and no *future* to introspect was provided') future = tasks.current_task(loop=loop) @@ -133,7 +133,7 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: f"with asyncio.Future" ) - call_stack: list[FrameCallStackEntry | CoroutineCallStackEntry] = [] + call_graph: list[FrameCallGraphEntry | CoroutineCallGraphEntry] = [] f = sys._getframe(1) try: @@ -141,13 +141,13 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: is_async = f.f_generator is not None if is_async: - call_stack.append(CoroutineCallStackEntry(f.f_generator)) + call_graph.append(CoroutineCallGraphEntry(f.f_generator)) if f.f_back is not None and f.f_back.f_generator is None: # We've reached the bottom of the coroutine stack, which # must be the Task that runs it. break else: - call_stack.append(FrameCallStackEntry(f)) + call_graph.append(FrameCallGraphEntry(f)) f = f.f_back finally: @@ -158,13 +158,13 @@ def capture_call_stack(*, future: any = None) -> FutureCallStack | None: for parent in fut_waiters: awaited_by.append(_build_stack_for_future(parent)) - return FutureCallStack(future, call_stack, awaited_by) + return FutureCallGraph(future, call_graph, awaited_by) -def print_call_stack(*, future: any = None, file=None) -> None: +def print_call_graph(*, future: any = None, file=None) -> None: """Print async call stack for the current task or the provided Future.""" - def render_level(st: FutureCallStack, buf: list[str], level: int): + def render_level(st: FutureCallGraph, buf: list[str], level: int): def add_line(line: str): buf.append(level * ' ' + line) @@ -177,12 +177,12 @@ def add_line(line: str): f'* Future(id=0x{id(st.future):x})' ) - if st.call_stack: + if st.call_graph: add_line( f' + Call stack:' ) - for ste in st.call_stack: - if isinstance(ste, FrameCallStackEntry): + for ste in st.call_graph: + if isinstance(ste, FrameCallGraphEntry): f = ste.frame add_line( f' | * {f.f_code.co_qualname}()' @@ -191,7 +191,7 @@ def add_line(line: str): f' | {f.f_code.co_filename}:{f.f_lineno}' ) else: - assert isinstance(ste, CoroutineCallStackEntry) + assert isinstance(ste, CoroutineCallGraphEntry) c = ste.coroutine try: @@ -222,7 +222,7 @@ def add_line(line: str): for fut in st.awaited_by: render_level(fut, buf, level + 1) - stack = capture_call_stack(future=future) + stack = capture_call_graph(future=future) if stack is None: return diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index 7f434b242f5e442..6d551b32525a5e1 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -20,13 +20,13 @@ def walk(s): [ ( f"s {entry.frame.f_code.co_name}" - if isinstance(entry, asyncio.FrameCallStackEntry) else + if isinstance(entry, asyncio.FrameCallGraphEntry) else ( f"a {entry.coroutine.cr_code.co_name}" if hasattr(entry.coroutine, 'cr_code') else f"ag {entry.coroutine.ag_code.co_name}" ) - ) for entry in s.call_stack + ) for entry in s.call_graph ] ) @@ -39,9 +39,9 @@ def walk(s): return ret buf = io.StringIO() - asyncio.print_call_stack(future=fut, file=buf) + asyncio.print_call_graph(future=fut, file=buf) - stack = asyncio.capture_call_stack(future=fut) + stack = asyncio.capture_call_graph(future=fut) return walk(stack), buf.getvalue() From fd141d41eeab7648c45a189aca7c9016ab3dfe88 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 11:17:28 -0700 Subject: [PATCH 17/84] Add NEWS --- .../next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst diff --git a/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst b/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst new file mode 100644 index 000000000000000..071dd9695d4566e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst @@ -0,0 +1 @@ +Add asyncio.capture_call_graph() and asyncio.print_call_graph() functions. From 2d72f2402b83485a535e136365a7429ad22e78b7 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 11:31:56 -0700 Subject: [PATCH 18/84] Per mpage's suggestion: use traceback style formatring for frames --- Doc/library/asyncio-stack.rst | 31 ++++++++++++++++------------- Lib/asyncio/stack.py | 30 +++++++++++++++++----------- Lib/test/test_asyncio/test_stack.py | 2 +- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 675422354e5e92c..fc523d0a7f10a06 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -18,7 +18,7 @@ a suspended *future*. .. versionadded:: 3.14 -.. function:: print_call_graph(*, future=None, file=None) +.. function:: print_call_graph(*, future=None, file=None, depth=1) Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. @@ -27,6 +27,10 @@ a suspended *future*. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. + If the function is called on *the current task*, the optional + keyword-only ``depth`` argument can be used to skip the specified + number of frames from top of the stack. + If *file* is not specified the function will print to :data:`sys.stdout`. **Example:** @@ -48,19 +52,14 @@ a suspended *future*. will print:: - * Task(name='Task-2', id=0x105038fe0) - + Call stack: - | * print_call_graph() - | asyncio/stack.py:231 - | * async test() - | test.py:4 - + Awaited by: - * Task(name='Task-1', id=0x1050a6060) - + Call stack: - | * async TaskGroup.__aexit__() - | asyncio/taskgroups.py:107 - | * async main() - | test.py:7 + * Task(name='Task-2', id=0x1039f0fe0) + + Call stack: + | File 't2.py', line 4, in async test() + + Awaited by: + * Task(name='Task-1', id=0x103a5e060) + + Call stack: + | File 'taskgroups.py', line 107, in async TaskGroup.__aexit__() + | File 't2.py', line 7, in async main() For rendering the call stack to a string the following pattern should be used: @@ -85,6 +84,10 @@ a suspended *future*. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. + If the function is called on *the current task*, the optional + keyword-only ``depth`` argument can be used to skip the specified + number of frames from top of the stack. + Returns a ``FutureCallGraph`` named tuple: * ``FutureCallGraph(future, call_graph, awaited_by)`` diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 150638365360ff9..c8c7e39fc5aa0d8 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -75,7 +75,11 @@ def _build_stack_for_future(future: any) -> FutureCallGraph: return FutureCallGraph(future, st, awaited_by) -def capture_call_graph(*, future: any = None) -> FutureCallGraph | None: +def capture_call_graph( + *, + future: any = None, + depth: int = 1, +) -> FutureCallGraph | None: """Capture async call stack for the current task or the provided Future. The stack is represented with three data structures: @@ -103,6 +107,10 @@ def capture_call_graph(*, future: any = None) -> FutureCallGraph | None: Receives an optional keyword-only "future" argument. If not passed, the current task will be used. If there's no current task, the function returns None. + + If "capture_call_graph()" is introspecting *the current task*, the + optional keyword-only "depth" argument can be used to skip the specified + number of frames from top of the stack. """ loop = events._get_running_loop() @@ -135,7 +143,7 @@ def capture_call_graph(*, future: any = None) -> FutureCallGraph | None: call_graph: list[FrameCallGraphEntry | CoroutineCallGraphEntry] = [] - f = sys._getframe(1) + f = sys._getframe(depth) try: while f is not None: is_async = f.f_generator is not None @@ -161,7 +169,7 @@ def capture_call_graph(*, future: any = None) -> FutureCallGraph | None: return FutureCallGraph(future, call_graph, awaited_by) -def print_call_graph(*, future: any = None, file=None) -> None: +def print_call_graph(*, future: any = None, file=None, depth: int = 1) -> None: """Print async call stack for the current task or the provided Future.""" def render_level(st: FutureCallGraph, buf: list[str], level: int): @@ -185,10 +193,9 @@ def add_line(line: str): if isinstance(ste, FrameCallGraphEntry): f = ste.frame add_line( - f' | * {f.f_code.co_qualname}()' - ) - add_line( - f' | {f.f_code.co_filename}:{f.f_lineno}' + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {f.f_code.co_qualname}()' ) else: assert isinstance(ste, CoroutineCallGraphEntry) @@ -209,10 +216,9 @@ def add_line(line: str): tag = 'generator' add_line( - f' | * {tag} {code.co_qualname}()' - ) - add_line( - f' | {f.f_code.co_filename}:{f.f_lineno}' + f' | File {f.f_code.co_filename!r},' + f' line {f.f_lineno}, in' + f' {tag} {code.co_qualname}()' ) if st.awaited_by: @@ -222,7 +228,7 @@ def add_line(line: str): for fut in st.awaited_by: render_level(fut, buf, level + 1) - stack = capture_call_graph(future=future) + stack = capture_call_graph(future=future, depth=depth + 1) if stack is None: return diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index 6d551b32525a5e1..f9244aab9b40527 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -107,7 +107,7 @@ async def main(): ]) self.assertIn( - '* async TestCallStack.test_stack_tgroup()', + ' async TestCallStack.test_stack_tgroup()', stack_for_c5[1]) From 391defa235f3d29cf4fef76e295ef8dd0fef32a1 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 11:35:50 -0700 Subject: [PATCH 19/84] mpage feedback: fix typo --- Doc/library/asyncio-stack.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index fc523d0a7f10a06..02eea8ffc64b696 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -23,7 +23,7 @@ a suspended *future*. Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. - The function recieves an optional keyword-only *future* argument. + The function receives an optional keyword-only *future* argument. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. @@ -80,7 +80,7 @@ a suspended *future*. Capture the async call graph for the current task or the provided :class:`Task` or :class:`Future`. - The function recieves an optional keyword-only *future* argument. + The function receives an optional keyword-only *future* argument. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. From d6357fdad1553273c35c46f6a5bc2f70fbcc2a77 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 11:47:06 -0700 Subject: [PATCH 20/84] Per mpage suggestion: get rid of CoroutineCallGraphEntry --- Doc/library/asyncio-stack.rst | 10 ++----- Lib/asyncio/stack.py | 42 +++++++++++------------------ Lib/test/test_asyncio/test_stack.py | 10 +++---- 3 files changed, 22 insertions(+), 40 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 02eea8ffc64b696..285a86fb1fbeb55 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -90,13 +90,12 @@ a suspended *future*. Returns a ``FutureCallGraph`` named tuple: - * ``FutureCallGraph(future, call_graph, awaited_by)`` + * ``FutureCallGraph(future, call_stack, awaited_by)`` Where 'future' is a reference to a *Future* or a *Task* (or their subclasses.) - ``call_graph`` is a list of ``FrameCallGraphEntry`` and - ``CoroutineCallGraphEntry`` objects (more on them below.) + ``call_stack`` is a list of ``FrameCallGraphEntry`` objects. ``awaited_by`` is a list of ``FutureCallGraph`` tuples. @@ -105,11 +104,6 @@ a suspended *future*. Where ``frame`` is a frame object of a regular Python function in the call stack. - * ``CoroutineCallGraphEntry(coroutine)`` - - Where ``coroutine`` is a coroutine object of an awaiting coroutine - or asyncronous generator. - Low level utility functions =========================== diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index c8c7e39fc5aa0d8..97a1b209aa6e007 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -12,7 +12,6 @@ 'capture_call_graph', 'print_call_graph', 'FrameCallGraphEntry', - 'CoroutineCallGraphEntry', 'FutureCallGraph', ) @@ -28,13 +27,9 @@ class FrameCallGraphEntry(typing.NamedTuple): frame: types.FrameType -class CoroutineCallGraphEntry(typing.NamedTuple): - coroutine: types.CoroutineType - - class FutureCallGraph(typing.NamedTuple): future: futures.Future - call_graph: list[FrameCallGraphEntry | CoroutineCallGraphEntry] + call_stack: list[FrameCallGraphEntry] awaited_by: list[FutureCallGraph] @@ -52,17 +47,17 @@ def _build_stack_for_future(future: any) -> FutureCallGraph: else: coro = get_coro() - st: list[CoroutineCallGraphEntry] = [] + st: list[FrameCallGraphEntry] = [] awaited_by: list[FutureCallGraph] = [] while coro is not None: if hasattr(coro, 'cr_await'): # A native coroutine or duck-type compatible iterator - st.append(CoroutineCallGraphEntry(coro)) + st.append(FrameCallGraphEntry(coro.cr_frame)) coro = coro.cr_await elif hasattr(coro, 'ag_await'): # A native async generator or duck-type compatible iterator - st.append(CoroutineCallGraphEntry(coro)) + st.append(FrameCallGraphEntry(coro.cr_frame)) coro = coro.ag_await else: break @@ -84,13 +79,12 @@ def capture_call_graph( The stack is represented with three data structures: - * FutureCallGraph(future, call_graph, awaited_by) + * FutureCallGraph(future, call_stack, awaited_by) Where 'future' is a reference to an asyncio.Future or asyncio.Task (or their subclasses.) - 'call_graph' is a list of FrameCallGraphEntry and CoroutineCallGraphEntry - objects (more on them below.) + 'call_stack' is a list of FrameGraphEntry objects. 'awaited_by' is a list of FutureCallGraph objects. @@ -99,11 +93,6 @@ def capture_call_graph( Where 'frame' is a frame object of a regular Python function in the call stack. - * CoroutineCallGraphEntry(coroutine) - - Where 'coroutine' is a coroutine object of an awaiting coroutine - or asyncronous generator. - Receives an optional keyword-only "future" argument. If not passed, the current task will be used. If there's no current task, the function returns None. @@ -141,21 +130,19 @@ def capture_call_graph( f"with asyncio.Future" ) - call_graph: list[FrameCallGraphEntry | CoroutineCallGraphEntry] = [] + call_stack: list[FrameCallGraphEntry] = [] f = sys._getframe(depth) try: while f is not None: is_async = f.f_generator is not None + call_stack.append(FrameCallGraphEntry(f)) if is_async: - call_graph.append(CoroutineCallGraphEntry(f.f_generator)) if f.f_back is not None and f.f_back.f_generator is None: # We've reached the bottom of the coroutine stack, which # must be the Task that runs it. break - else: - call_graph.append(FrameCallGraphEntry(f)) f = f.f_back finally: @@ -166,7 +153,7 @@ def capture_call_graph( for parent in fut_waiters: awaited_by.append(_build_stack_for_future(parent)) - return FutureCallGraph(future, call_graph, awaited_by) + return FutureCallGraph(future, call_stack, awaited_by) def print_call_graph(*, future: any = None, file=None, depth: int = 1) -> None: @@ -185,12 +172,14 @@ def add_line(line: str): f'* Future(id=0x{id(st.future):x})' ) - if st.call_graph: + if st.call_stack: add_line( f' + Call stack:' ) - for ste in st.call_graph: - if isinstance(ste, FrameCallGraphEntry): + for ste in st.call_stack: + f = ste.frame + + if f.f_generator is None: f = ste.frame add_line( f' | File {f.f_code.co_filename!r},' @@ -198,8 +187,7 @@ def add_line(line: str): f' {f.f_code.co_qualname}()' ) else: - assert isinstance(ste, CoroutineCallGraphEntry) - c = ste.coroutine + c = f.f_generator try: f = c.cr_frame diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index f9244aab9b40527..9bb675270e4be61 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -20,13 +20,13 @@ def walk(s): [ ( f"s {entry.frame.f_code.co_name}" - if isinstance(entry, asyncio.FrameCallGraphEntry) else + if entry.frame.f_generator is None else ( - f"a {entry.coroutine.cr_code.co_name}" - if hasattr(entry.coroutine, 'cr_code') else - f"ag {entry.coroutine.ag_code.co_name}" + f"a {entry.frame.f_generator.cr_code.co_name}" + if hasattr(entry.frame.f_generator, 'cr_code') else + f"ag {entry.frame.f_generator.ag_code.co_name}" ) - ) for entry in s.call_graph + ) for entry in s.call_stack ] ) From bb3b6df709f146d88eff702ed27e79f858dda463 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 12:06:30 -0700 Subject: [PATCH 21/84] Strip sloppy white space! --- Include/internal/pycore_runtime.h | 2 +- Modules/_testexternalinspection.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index b51f427dfa15795..f9bbabfce2f2321 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -70,7 +70,7 @@ typedef struct _Py_AuditHookEntry { _GENERATE_DEBUG_SECTION_WINDOWS(name) \ _GENERATE_DEBUG_SECTION_APPLE(name) \ declaration \ - _GENERATE_DEBUG_SECTION_LINUX(name) + _GENERATE_DEBUG_SECTION_LINUX(name) #if defined(MS_WINDOWS) #define _GENERATE_DEBUG_SECTION_WINDOWS(name) \ diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 6a14c399933b056..2cd59135406475e 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -557,7 +557,7 @@ read_py_long( _Py_DebugOffsets* offsets, uintptr_t address) { unsigned int shift = PYLONG_BITS_IN_DIGIT; - + ssize_t size; uintptr_t lv_tag; int bytes_read = read_memory(pid, address + offsets->long_object.lv_tag, sizeof(uintptr_t), &lv_tag); @@ -581,7 +581,7 @@ read_py_long( } long value = 0; - + for (ssize_t i = 0; i < size; ++i) { long long factor; if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), &factor)) { @@ -642,7 +642,7 @@ parse_task_name( } return PyUnicode_FromFormat("Task-%d", res); } - + if(!(flags & Py_TPFLAGS_UNICODE_SUBCLASS)) { PyErr_SetString(PyExc_RuntimeError, "Invalid task name object"); return NULL; From 54c99ec94246d24f54385ab37d029fde86273666 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 12:48:35 -0700 Subject: [PATCH 22/84] Fix sloppy set iteration! --- Include/internal/pycore_runtime.h | 1 + Include/internal/pycore_runtime_init.h | 1 + Modules/_testexternalinspection.c | 22 ++++++++++++++++++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index f9bbabfce2f2321..0b12132a3edf10e 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -193,6 +193,7 @@ typedef struct _Py_DebugOffsets { uint64_t size; uint64_t used; uint64_t table; + uint64_t mask; } set_object; // PyDict object offset; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 1072196abe4fc91..04c09f5e96067da 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -112,6 +112,7 @@ extern PyTypeObject _PyExc_MemoryError; .size = sizeof(PySetObject), \ .used = offsetof(PySetObject, used), \ .table = offsetof(PySetObject, table), \ + .mask = offsetof(PySetObject, mask), \ }, \ .dict_object = { \ .size = sizeof(PyDictObject), \ diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 2cd59135406475e..daf7aad6804452c 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -910,16 +910,25 @@ parse_tasks_in_set( return -1; } - Py_ssize_t set_len; + Py_ssize_t num_els; if (read_ssize_t( pid, set_obj + offsets->set_object.used, + &num_els) + ) { + return -1; + } + + Py_ssize_t set_len; + if (read_ssize_t( + pid, + set_obj + offsets->set_object.mask, &set_len) ) { return -1; } + set_len++; // The set contains the `mask+1` element slots. - Py_ssize_t cnt = 0; uintptr_t table_ptr; if (read_ptr( pid, @@ -929,7 +938,9 @@ parse_tasks_in_set( return -1; } - while (cnt < set_len) { + Py_ssize_t i = 0; + Py_ssize_t els = 0; + while (i < set_len) { uintptr_t key_addr; if (read_py_ptr(pid, table_ptr, &key_addr)) { return -1; @@ -954,11 +965,14 @@ parse_tasks_in_set( return -1; } - cnt++; + if (++els == num_els) { + break; + } } } table_ptr += sizeof(void*) * 2; + i++; } return 0; } From c1a4f09b45c27b526e4a21fad35c9403029e69ca Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 12:56:26 -0700 Subject: [PATCH 23/84] Fix a sloppy test! --- Lib/test/test_external_inspection.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index d7f55fc6b068415..227ac574e9631d0 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -134,14 +134,16 @@ async def main(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) expected_stack_trace = [ ["c5", "c4", "c3", "c2"], "c2_root", [ [["main"], "Task-1", []], - [["c1"], "sub_main_2", [[["main"], "Task-1", []]]], [["c1"], "sub_main_1", [[["main"], "Task-1", []]]], + [["c1"], "sub_main_2", [[["main"], "Task-1", []]]], ], ] self.assertEqual(stack_trace, expected_stack_trace) From 027d5229a91ef4b0f1ea092f13543acf2be6bd0f Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 13:02:29 -0700 Subject: [PATCH 24/84] Run 'make regen-all' --- Include/internal/pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - Include/internal/pycore_runtime_init_generated.h | 1 - Include/internal/pycore_unicodeobject_generated.h | 4 ---- 4 files changed, 7 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 98a48bce511be44..28a76c36801b4bd 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -741,7 +741,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_asyncio_future_blocking)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_awaited_by)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_blksize)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_bootstrap)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_check_retval_)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index eddea908ed9709d..ac789b06fb8a616 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -230,7 +230,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) STRUCT_FOR_ID(_asyncio_future_blocking) - STRUCT_FOR_ID(_awaited_by) STRUCT_FOR_ID(_blksize) STRUCT_FOR_ID(_bootstrap) STRUCT_FOR_ID(_check_retval_) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 3f23898566c6d5c..7847a5c63ebf3f1 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -739,7 +739,6 @@ extern "C" { INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ INIT_ID(_asyncio_future_blocking), \ - INIT_ID(_awaited_by), \ INIT_ID(_blksize), \ INIT_ID(_bootstrap), \ INIT_ID(_check_retval_), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 87f7c090f57e03c..a688f70a2ba36ff 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -720,10 +720,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(_awaited_by); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_blksize); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); From c2d5ec6b20be4b44baf5abcedb93fb2030126aab Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 13:05:03 -0700 Subject: [PATCH 25/84] Add a what's new entry --- Doc/whatsnew/3.14.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 67d8d389b580829..ea6557f4d8fa271 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -385,6 +385,11 @@ asyncio reduces memory usage. (Contributed by Kumar Aditya in :gh:`107803`.) +* :mod:`asyncio` has new utility functions for introspecting and printing + the program's call graph. + (Contributed by Yury Selivanov and Pablo Galindo Salgado in :gh:`91048`.) + + Deprecated ========== From 08d09eb87151d9bd32f28423eabd439f16ce8e37 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 13:11:56 -0700 Subject: [PATCH 26/84] Fix (hopefully) a compiler warning --- Modules/_asynciomodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 032a7a50cf9c797..20fbfff46a9de3e 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -657,7 +657,7 @@ future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) } static PyObject * -future_get_awaited_by(FutureObj *fut) +FutureObj_get_awaited_by(FutureObj *fut, void *Py_UNUSED(ignored)) { /* Implementation of a Python getter. */ if (fut->fut_awaited_by == NULL) { @@ -1670,7 +1670,7 @@ static PyMethodDef FutureType_methods[] = { NULL, NULL}, \ {"_cancel_message", (getter)FutureObj_get_cancel_message, \ (setter)FutureObj_set_cancel_message, NULL}, \ - {"_asyncio_awaited_by", (getter)future_get_awaited_by, NULL, NULL}, + {"_asyncio_awaited_by", (getter)FutureObj_get_awaited_by, NULL, NULL}, static PyGetSetDef FutureType_getsetlist[] = { FUTURE_COMMON_GETSETLIST From fe3113bce50f8da8209be3e6cdd5c31bd19fd77b Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 13:36:33 -0700 Subject: [PATCH 27/84] Fix sloppy what's new! --- Doc/whatsnew/3.14.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index ea6557f4d8fa271..cf7bba1bf42d2e7 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -386,7 +386,8 @@ asyncio (Contributed by Kumar Aditya in :gh:`107803`.) * :mod:`asyncio` has new utility functions for introspecting and printing - the program's call graph. + the program's call graph: :func:`asyncio.capture_call_graph` and + :func:`asyncio.print_call_graph`. (Contributed by Yury Selivanov and Pablo Galindo Salgado in :gh:`91048`.) From 18ec26da2a316d02347085d7b69414525c713836 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 14:16:47 -0700 Subject: [PATCH 28/84] Fix CI complaining about suspicious global in C --- Tools/c-analyzer/cpython/ignored.tsv | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index e6c599a2ac4a464..c36c06e9d9fcf44 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -53,6 +53,9 @@ Python/pyhash.c - _Py_HashSecret - ## thread-safe hashtable (internal locks) Python/parking_lot.c - buckets - +## data needed for introspecting asyncio state from debuggers and profilers +Modules/_asynciomodule.c - AsyncioDebug - + ################################## ## state tied to Py_Main() From e4cc46245cb705f20729168983cfa5d14b22d194 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 2 Oct 2024 14:43:10 -0700 Subject: [PATCH 29/84] Add a test for depth=2 --- Lib/test/test_asyncio/test_stack.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index 9bb675270e4be61..62ec000b1c62f4e 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -8,7 +8,7 @@ def tearDownModule(): asyncio.set_event_loop_policy(None) -def capture_test_stack(*, fut=None): +def capture_test_stack(*, fut=None, depth=1): def walk(s): ret = [ @@ -39,9 +39,9 @@ def walk(s): return ret buf = io.StringIO() - asyncio.print_call_graph(future=fut, file=buf) + asyncio.print_call_graph(future=fut, file=buf, depth=depth+1) - stack = asyncio.capture_call_graph(future=fut) + stack = asyncio.capture_call_graph(future=fut, depth=depth) return walk(stack), buf.getvalue() @@ -54,7 +54,7 @@ async def test_stack_tgroup(self): def c5(): nonlocal stack_for_c5 - stack_for_c5 = capture_test_stack() + stack_for_c5 = capture_test_stack(depth=2) async def c4(): await asyncio.sleep(0) @@ -81,7 +81,7 @@ async def main(): # task name 'T', # call stack - ['s capture_test_stack', 's c5', 'a c4', 'a c3', 'a c2'], + ['s c5', 'a c4', 'a c3', 'a c2'], # awaited by [ ['T', From d5cdc36181809235cc6b2b38f86946075eddb6d9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 2 Oct 2024 22:43:54 +0100 Subject: [PATCH 30/84] Add critical sections for free threaded builds --- Modules/_asynciomodule.c | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 20fbfff46a9de3e..4acbb6c30644ffe 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -579,15 +579,8 @@ future_init(FutureObj *fut, PyObject *loop) } static int -future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) +future_awaited_by_add_lock_held(asyncio_state *state, PyObject *fut, PyObject *thing) { - if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { - // We only want to support native asyncio Futures. - // For further insight see the comment in the Python - // implementation of "future_add_to_awaited_by()". - return 0; - } - FutureObj *_fut = (FutureObj *)fut; /* Most futures/task are only awaited by one entity, so we want @@ -623,7 +616,7 @@ future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) } static int -future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) +future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) { if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { // We only want to support native asyncio Futures. @@ -632,6 +625,17 @@ future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) return 0; } + int result; + Py_BEGIN_CRITICAL_SECTION(fut); + result = future_awaited_by_add_lock_held(state, fut, thing); + Py_END_CRITICAL_SECTION(); + return result; +} + +static int +future_awaited_by_discard_lock_held(asyncio_state *state, PyObject *fut, PyObject *thing) +{ + FutureObj *_fut = (FutureObj *)fut; /* Following the semantics of 'set.discard()' here in not @@ -656,6 +660,23 @@ future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) return 0; } +static int +future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) +{ + if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { + // We only want to support native asyncio Futures. + // For further insight see the comment in the Python + // implementation of "future_add_to_awaited_by()". + return 0; + } + + int result; + Py_BEGIN_CRITICAL_SECTION(fut); + result = future_awaited_by_discard_lock_held(state, fut, thing); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * FutureObj_get_awaited_by(FutureObj *fut, void *Py_UNUSED(ignored)) { From 83606f2b9686e879058934c427cb57de834ab28f Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 2 Oct 2024 22:54:06 +0100 Subject: [PATCH 31/84] Add more slopy tests --- Lib/test/test_external_inspection.py | 111 +++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 227ac574e9631d0..c4c3bbe54ac19fe 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -148,7 +148,118 @@ async def main(): ] self.assertEqual(stack_trace, expected_stack_trace) + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_asyncgen_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import asyncio + import time + import os + import sys + import test.test_asyncio.test_stack as ts + + async def gen_nested_call(): + fifo = sys.argv[1] + with open(sys.argv[1], "w") as fifo: + fifo.write("ready") + time.sleep(10000) + + async def gen(): + for num in range(2): + yield num + if num == 1: + await gen_nested_call() + + async def main(): + async for el in gen(): + pass + + asyncio.run(main()) + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + expected_stack_trace = [['gen_nested_call', 'gen', 'main'], 'Task-1', []] + self.assertEqual(stack_trace, expected_stack_trace) + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + def test_async_gather_remote_stack_trace(self): + # Spawn a process with some realistic Python code + script = textwrap.dedent("""\ + import asyncio + import time + import os + import sys + import test.test_asyncio.test_stack as ts + + async def deep(): + await asyncio.sleep(0) + fifo = sys.argv[1] + with open(sys.argv[1], "w") as fifo: + fifo.write("ready") + time.sleep(10000) + + async def c1(): + await asyncio.sleep(0) + await deep() + + async def c2(): + await asyncio.sleep(0) + + async def main(): + await asyncio.gather(c1(), c2()) + + asyncio.run(main()) + """) + stack_trace = None + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + expected_stack_trace = [['deep', 'c1'], 'Task-2', [[['main'], 'Task-1', []]]] + self.assertEqual(stack_trace, expected_stack_trace) @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") From 5edac41fc485eac7fa1f3127f9b999b95c8c2675 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 10:37:08 -0700 Subject: [PATCH 32/84] Apply suggestions from code review Co-authored-by: Kumar Aditya --- Doc/library/asyncio-future.rst | 2 +- Doc/library/asyncio-stack.rst | 2 +- Lib/asyncio/futures.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index f1f23b8021b29f8..9dce07314119401 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -65,7 +65,7 @@ Future Functions and *loop* is not specified and there is no running event loop. -.. function:: wrap_future(future, /, *, loop=None) +.. function:: wrap_future(future, *, loop=None) Wrap a :class:`concurrent.futures.Future` object in a :class:`asyncio.Future` object. diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 285a86fb1fbeb55..1553e3fd0a891e3 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -28,7 +28,7 @@ a suspended *future*. current task, the function returns ``None``. If the function is called on *the current task*, the optional - keyword-only ``depth`` argument can be used to skip the specified + keyword-only *depth* argument can be used to skip the specified number of frames from top of the stack. If *file* is not specified the function will print to :data:`sys.stdout`. diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 5785477248a8d9f..ba16ae8e154b5ad 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -426,7 +426,7 @@ def wrap_future(future, *, loop=None): def future_add_to_awaited_by(fut, waiter, /): """Record that `fut` is awaited on by `waiter`.""" # For the sake of keeping the implementation minimal and assuming - # that 99.9% of asyncio users use the built-in Futures and Tasks + # that most of asyncio users use the built-in Futures and Tasks # (or their subclasses), we only support native Future objects # and their subclasses. # From 8dc6d340e906b6e6fc896816a253229b1ddfc2e7 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 10:44:40 -0700 Subject: [PATCH 33/84] Apply suggestions from code review --- Modules/_asynciomodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 4acbb6c30644ffe..e51a9bd1d6a38c5 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -649,7 +649,7 @@ future_awaited_by_discard_lock_held(asyncio_state *state, PyObject *fut, PyObjec return 0; } if (_fut->fut_awaited_by_is_set) { - assert(PySet_Check(_fut->fut_awaited_by)); + assert(PySet_CheckExact(_fut->fut_awaited_by)); int err = PySet_Discard(_fut->fut_awaited_by, thing); if (err < 0 && PyErr_Occurred()) { return -1; @@ -686,7 +686,7 @@ FutureObj_get_awaited_by(FutureObj *fut, void *Py_UNUSED(ignored)) } if (fut->fut_awaited_by_is_set) { /* Already a set, just wrap it into a frozen set and return. */ - assert(PySet_Check(fut->fut_awaited_by)); + assert(PySet_CheckExact(fut->fut_awaited_by)); return PyFrozenSet_New(fut->fut_awaited_by); } From 30884ea949c4f74833de441478d2ccdc98c01d33 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 11:09:25 -0700 Subject: [PATCH 34/84] Update Modules/_asynciomodule.c --- Modules/_asynciomodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index e51a9bd1d6a38c5..ded7990d7bf98af 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -651,7 +651,7 @@ future_awaited_by_discard_lock_held(asyncio_state *state, PyObject *fut, PyObjec if (_fut->fut_awaited_by_is_set) { assert(PySet_CheckExact(_fut->fut_awaited_by)); int err = PySet_Discard(_fut->fut_awaited_by, thing); - if (err < 0 && PyErr_Occurred()) { + if (err < 0) { return -1; } else { return 0; From 131765849eabb1722b0edf4aacf6a1d8c032a6f4 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 11:10:19 -0700 Subject: [PATCH 35/84] Update Modules/_asynciomodule.c Co-authored-by: Kumar Aditya --- Modules/_asynciomodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index ded7990d7bf98af..6f51109e972db38 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -594,7 +594,7 @@ future_awaited_by_add_lock_held(asyncio_state *state, PyObject *fut, PyObject *t } if (_fut->fut_awaited_by_is_set) { - assert(PySet_Check(_fut->fut_awaited_by)); + assert(PySet_CheckExact(_fut->fut_awaited_by)); return PySet_Add(_fut->fut_awaited_by, thing); } From 81b0a310cbdcf60fc32dfa779e534bc34715e8fa Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 11:20:26 -0700 Subject: [PATCH 36/84] Use dataclasses instead of tuples for asyncio.stack --- Doc/library/asyncio-stack.rst | 4 ++-- Lib/asyncio/stack.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 1553e3fd0a891e3..5263522cca0b717 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -88,7 +88,7 @@ a suspended *future*. keyword-only ``depth`` argument can be used to skip the specified number of frames from top of the stack. - Returns a ``FutureCallGraph`` named tuple: + Returns a ``FutureCallGraph`` data class object: * ``FutureCallGraph(future, call_stack, awaited_by)`` @@ -97,7 +97,7 @@ a suspended *future*. ``call_stack`` is a list of ``FrameCallGraphEntry`` objects. - ``awaited_by`` is a list of ``FutureCallGraph`` tuples. + ``awaited_by`` is a list of ``FutureCallGraph`` objects. * ``FrameCallGraphEntry(frame)`` diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 97a1b209aa6e007..dbb92fd5d5ff23a 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -1,8 +1,8 @@ """Introspection utils for tasks call stacks.""" +import dataclasses import sys import types -import typing from . import events from . import futures @@ -23,11 +23,13 @@ # top level asyncio namespace, and want to avoid future name clashes. -class FrameCallGraphEntry(typing.NamedTuple): +@dataclasses.dataclass(frozen=True) +class FrameCallGraphEntry: frame: types.FrameType -class FutureCallGraph(typing.NamedTuple): +@dataclasses.dataclass(frozen=True) +class FutureCallGraph: future: futures.Future call_stack: list[FrameCallGraphEntry] awaited_by: list[FutureCallGraph] From 258ce3dcc36645fb88693ab99a035aac4b3ae970 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 11:21:01 -0700 Subject: [PATCH 37/84] Fix sloppy whitespace! --- Modules/_asynciomodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 6f51109e972db38..7700f1b7a0f56f0 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -16,6 +16,7 @@ #include // offsetof() + /*[clinic input] module _asyncio [clinic start generated code]*/ From b9ecefb0477e12c70ead97bc628513cf12f07f43 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 16 Oct 2024 19:28:29 +0100 Subject: [PATCH 38/84] Remove critical sections for now --- Modules/_asynciomodule.c | 39 +++++++++------------------------------ 1 file changed, 9 insertions(+), 30 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 7700f1b7a0f56f0..f684f5cf4a1ca77 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -580,8 +580,15 @@ future_init(FutureObj *fut, PyObject *loop) } static int -future_awaited_by_add_lock_held(asyncio_state *state, PyObject *fut, PyObject *thing) +future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) { + if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { + // We only want to support native asyncio Futures. + // For further insight see the comment in the Python + // implementation of "future_add_to_awaited_by()". + return 0; + } + FutureObj *_fut = (FutureObj *)fut; /* Most futures/task are only awaited by one entity, so we want @@ -617,7 +624,7 @@ future_awaited_by_add_lock_held(asyncio_state *state, PyObject *fut, PyObject *t } static int -future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) +future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) { if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { // We only want to support native asyncio Futures. @@ -626,17 +633,6 @@ future_awaited_by_add(asyncio_state *state, PyObject *fut, PyObject *thing) return 0; } - int result; - Py_BEGIN_CRITICAL_SECTION(fut); - result = future_awaited_by_add_lock_held(state, fut, thing); - Py_END_CRITICAL_SECTION(); - return result; -} - -static int -future_awaited_by_discard_lock_held(asyncio_state *state, PyObject *fut, PyObject *thing) -{ - FutureObj *_fut = (FutureObj *)fut; /* Following the semantics of 'set.discard()' here in not @@ -661,23 +657,6 @@ future_awaited_by_discard_lock_held(asyncio_state *state, PyObject *fut, PyObjec return 0; } -static int -future_awaited_by_discard(asyncio_state *state, PyObject *fut, PyObject *thing) -{ - if (!TaskOrFuture_Check(state, fut) || !TaskOrFuture_Check(state, thing)) { - // We only want to support native asyncio Futures. - // For further insight see the comment in the Python - // implementation of "future_add_to_awaited_by()". - return 0; - } - - int result; - Py_BEGIN_CRITICAL_SECTION(fut); - result = future_awaited_by_discard_lock_held(state, fut, thing); - Py_END_CRITICAL_SECTION(); - return result; -} - static PyObject * FutureObj_get_awaited_by(FutureObj *fut, void *Py_UNUSED(ignored)) { From b77dcb0fbcdfd94976ac11c76a4063194e86217c Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 11:36:26 -0700 Subject: [PATCH 39/84] Eplain the weird macro --- Modules/_asynciomodule.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index f684f5cf4a1ca77..07f5783cbe77d2f 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -87,6 +87,8 @@ typedef struct { (Task_CheckExact(state, obj) \ || PyObject_TypeCheck(obj, state->TaskType)) +// This macro is optimized to quickly return for native Future *or* Task +// objects by inlining fast "exact" checks to be called first. #define TaskOrFuture_Check(state, obj) \ (Task_CheckExact(state, obj) \ || Future_CheckExact(state, obj) \ From 8867946e7e72d9c4c2cc5a9e71289986c0544cf9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 16 Oct 2024 19:55:30 +0100 Subject: [PATCH 40/84] Fix test --- Lib/test/test_asyncio/test_stack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_stack.py index 62ec000b1c62f4e..9536451e0efa858 100644 --- a/Lib/test/test_asyncio/test_stack.py +++ b/Lib/test/test_asyncio/test_stack.py @@ -85,13 +85,13 @@ async def main(): # awaited by [ ['T', - ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] ], ['T', ['a c1'], [ ['T', - ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] ] ] ], @@ -99,7 +99,7 @@ async def main(): ['a c1'], [ ['T', - ['a __aexit__', 'a main', 'a test_stack_tgroup'], [] + ['a _aexit', 'a __aexit__', 'a main', 'a test_stack_tgroup'], [] ] ] ] From b47bef111963903c57e62ad1179293fc5fe772a5 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 16 Oct 2024 12:26:28 -0700 Subject: [PATCH 41/84] Add a docs clarification --- Doc/library/asyncio-stack.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-stack.rst index 5263522cca0b717..fab72521bb9df8e 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-stack.rst @@ -13,7 +13,9 @@ Stack Introspection asyncio has powerful runtime call stack introspection utilities to trace the entire call graph of a running coroutine or task, or -a suspended *future*. +a suspended *future*. These utilities and the underlying machinery +can be used by users in their Python code or by external profilers +and debuggers. .. versionadded:: 3.14 From 230b7ecd69c9a319604055d734aafdc82ae244a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 16 Oct 2024 23:53:25 +0200 Subject: [PATCH 42/84] Fix typing in asyncio.stack --- Lib/asyncio/stack.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index dbb92fd5d5ff23a..451f0639e9f88b1 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -3,6 +3,7 @@ import dataclasses import sys import types +import typing from . import events from . import futures @@ -35,18 +36,15 @@ class FutureCallGraph: awaited_by: list[FutureCallGraph] -def _build_stack_for_future(future: any) -> FutureCallGraph: +def _build_stack_for_future(future: futures.Future) -> FutureCallGraph: if not isinstance(future, futures.Future): raise TypeError( f"{future!r} object does not appear to be compatible " f"with asyncio.Future" ) - try: - get_coro = future.get_coro - except AttributeError: - coro = None - else: + coro = None + if get_coro := getattr(future, 'get_coro', None): coro = get_coro() st: list[FrameCallGraphEntry] = [] @@ -74,7 +72,7 @@ def _build_stack_for_future(future: any) -> FutureCallGraph: def capture_call_graph( *, - future: any = None, + future: futures.Future | None = None, depth: int = 1, ) -> FutureCallGraph | None: """Capture async call stack for the current task or the provided Future. @@ -158,11 +156,16 @@ def capture_call_graph( return FutureCallGraph(future, call_stack, awaited_by) -def print_call_graph(*, future: any = None, file=None, depth: int = 1) -> None: +def print_call_graph( + *, + future: futures.Future | None = None, + file: typing.TextIO | None = None, + depth: int = 1, +) -> None: """Print async call stack for the current task or the provided Future.""" - def render_level(st: FutureCallGraph, buf: list[str], level: int): - def add_line(line: str): + def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None: + def add_line(line: str) -> None: buf.append(level * ' ' + line) if isinstance(st.future, tasks.Task): @@ -223,7 +226,7 @@ def add_line(line: str): return try: - buf = [] + buf: list[str] = [] render_level(stack, buf, 0) rendered = '\n'.join(buf) print(rendered, file=file) From b1d61582407563a3e3bc46954cd42720801a93ad Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 17 Oct 2024 01:03:39 +0100 Subject: [PATCH 43/84] Fix memory leak --- Modules/_testexternalinspection.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index daf7aad6804452c..390c5caf5b575b6 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -577,7 +577,7 @@ read_py_long( } bytes_read = read_memory(pid, address + offsets->long_object.ob_digit, sizeof(digit) * size, digits); if (bytes_read < 0) { - return -1; + goto error; } long value = 0; @@ -585,17 +585,20 @@ read_py_long( for (ssize_t i = 0; i < size; ++i) { long long factor; if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), &factor)) { - return -1; + goto error; } if (__builtin_add_overflow(value, factor, &value)) { - return -1; + goto error; } } + PyMem_RawFree(digits); if (negative) { value = -1 * value; } - return value; +error: + PyMem_RawFree(digits); + return -1; } static PyObject * From ac5136433de3ba6f58a0447886aef50710d0ab21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 17 Oct 2024 16:37:55 +0200 Subject: [PATCH 44/84] Address comments from Kumar's review --- Lib/asyncio/futures.py | 2 -- Lib/asyncio/stack.py | 16 ++++++++-------- ...2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst | 3 ++- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index c9cbcc6f6d81732..4e6d0cf39cd3c51 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -46,8 +46,6 @@ class Future: """ - _awaited_by = None - # Class variables serving as defaults for instance variables. _state = _PENDING _result = None diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/stack.py index 451f0639e9f88b1..f68e80f55063af0 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/stack.py @@ -24,12 +24,12 @@ # top level asyncio namespace, and want to avoid future name clashes. -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass(frozen=True, slots=True) class FrameCallGraphEntry: frame: types.FrameType -@dataclasses.dataclass(frozen=True) +@dataclasses.dataclass(frozen=True, slots=True) class FutureCallGraph: future: futures.Future call_stack: list[FrameCallGraphEntry] @@ -62,8 +62,8 @@ def _build_stack_for_future(future: futures.Future) -> FutureCallGraph: else: break - if fut_waiters := getattr(future, '_asyncio_awaited_by', None): - for parent in fut_waiters: + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: awaited_by.append(_build_stack_for_future(parent)) st.reverse() @@ -149,8 +149,8 @@ def capture_call_graph( del f awaited_by = [] - if fut_waiters := getattr(future, '_asyncio_awaited_by', None): - for parent in fut_waiters: + if future._asyncio_awaited_by: + for parent in future._asyncio_awaited_by: awaited_by.append(_build_stack_for_future(parent)) return FutureCallGraph(future, call_stack, awaited_by) @@ -170,11 +170,11 @@ def add_line(line: str) -> None: if isinstance(st.future, tasks.Task): add_line( - f'* Task(name={st.future.get_name()!r}, id=0x{id(st.future):x})' + f'* Task(name={st.future.get_name()!r}, id={id(st.future):#x})' ) else: add_line( - f'* Future(id=0x{id(st.future):x})' + f'* Future(id={id(st.future):#x})' ) if st.call_stack: diff --git a/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst b/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst index 071dd9695d4566e..c2faf470ffc9cf6 100644 --- a/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst +++ b/Misc/NEWS.d/next/Library/2024-10-02-11-17-23.gh-issue-91048.QWY-b1.rst @@ -1 +1,2 @@ -Add asyncio.capture_call_graph() and asyncio.print_call_graph() functions. +Add :func:`asyncio.capture_call_graph` and +:func:`asyncio.print_call_graph` functions. From c7e59eb5c599cd8bcebc443227d536ad5720b9c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 22 Oct 2024 21:23:07 +0200 Subject: [PATCH 45/84] Fix rare Mach-O linker bug when .bss is uninitialized data --- Lib/test/test_external_inspection.py | 18 ++++++++++-------- Modules/_testexternalinspection.c | 8 +++++++- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index c4c3bbe54ac19fe..3391b48554fc1d5 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -38,8 +38,8 @@ def baz(): foo() def foo(): - fifo = sys.argv[1] - with open(sys.argv[1], "w") as fifo: + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: fifo.write("ready") time.sleep(1000) @@ -87,8 +87,8 @@ def test_async_remote_stack_trace(self): import test.test_asyncio.test_stack as ts def c5(): - fifo = sys.argv[1] - with open(sys.argv[1], "w") as fifo: + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: fifo.write("ready") time.sleep(10000) @@ -128,6 +128,8 @@ async def main(): stack_trace = get_async_stack_trace(p.pid) except PermissionError: self.skipTest("Insufficient permissions to read the stack trace") + except RuntimeError: + breakpoint() finally: os.remove(fifo) p.kill() @@ -160,8 +162,8 @@ def test_asyncgen_remote_stack_trace(self): import test.test_asyncio.test_stack as ts async def gen_nested_call(): - fifo = sys.argv[1] - with open(sys.argv[1], "w") as fifo: + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: fifo.write("ready") time.sleep(10000) @@ -217,8 +219,8 @@ def test_async_gather_remote_stack_trace(self): async def deep(): await asyncio.sleep(0) - fifo = sys.argv[1] - with open(sys.argv[1], "w") as fifo: + fifo_path = sys.argv[1] + with open(fifo_path, "w") as fifo: fifo.write("ready") time.sleep(10000) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 390c5caf5b575b6..ddbc9dd53a69302 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -218,6 +218,12 @@ search_map_for_section(pid_t pid, const char* secname, const char* substr) { continue; } + if ((region_info.protection & VM_PROT_READ) == 0 + || (region_info.protection & VM_PROT_EXECUTE) == 0) { + address += size; + continue; + } + char* filename = strrchr(map_filename, '/'); if (filename != NULL) { filename++; // Move past the '/' @@ -1222,7 +1228,7 @@ read_async_debug( struct _Py_AsyncioModuleDebugOffsets* async_debug ) { uintptr_t async_debug_addr = get_async_debug(pid); - if (!async_debug) { + if (!async_debug_addr) { return -1; } size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); From 59121f64fee619eb06282af8473122bab0fabb2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 22 Oct 2024 22:32:20 +0200 Subject: [PATCH 46/84] you saw nothing, okay --- Lib/test/test_external_inspection.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 3391b48554fc1d5..6be41042e517d52 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -128,8 +128,6 @@ async def main(): stack_trace = get_async_stack_trace(p.pid) except PermissionError: self.skipTest("Insufficient permissions to read the stack trace") - except RuntimeError: - breakpoint() finally: os.remove(fifo) p.kill() From f8f48f0db7188e57a7ad9e6430bd9d4b23b1db1f Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Tue, 22 Oct 2024 22:18:21 -0700 Subject: [PATCH 47/84] Update Lib/asyncio/futures.py Co-authored-by: Savannah Ostrowski --- Lib/asyncio/futures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 4e6d0cf39cd3c51..8bd47a90c7a83a5 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -450,7 +450,7 @@ def future_add_to_awaited_by(fut, waiter, /): def future_discard_from_awaited_by(fut, waiter, /): """Record that `fut` is no longer awaited on by `waiter`.""" # See the comment in "future_add_to_awaited_by()" body for - # details on implemntation. + # details on implementation. # # Note that there's an accelerated version of this function # shadowing this implementation later in this file. From 74c5ad141ba6a365d99228374880e6b0ca63d863 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 30 Oct 2024 17:53:45 +0100 Subject: [PATCH 48/84] Remove unnecessary imports from tests --- Lib/test/test_external_inspection.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 6be41042e517d52..9ef3ea86dd002f1 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -29,7 +29,7 @@ class TestGetStackTrace(unittest.TestCase): def test_remote_stack_trace(self): # Spawn a process with some realistic Python code script = textwrap.dedent("""\ - import time, sys, os + import time, sys def bar(): for x in range(100): if x == 50: @@ -82,9 +82,7 @@ def test_async_remote_stack_trace(self): script = textwrap.dedent("""\ import asyncio import time - import os import sys - import test.test_asyncio.test_stack as ts def c5(): fifo_path = sys.argv[1] @@ -155,9 +153,7 @@ def test_asyncgen_remote_stack_trace(self): script = textwrap.dedent("""\ import asyncio import time - import os import sys - import test.test_asyncio.test_stack as ts async def gen_nested_call(): fifo_path = sys.argv[1] @@ -211,9 +207,7 @@ def test_async_gather_remote_stack_trace(self): script = textwrap.dedent("""\ import asyncio import time - import os import sys - import test.test_asyncio.test_stack as ts async def deep(): await asyncio.sleep(0) From 067c043cc4792fa19a04a905b16a2da3731a7ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 31 Oct 2024 23:36:09 +0100 Subject: [PATCH 49/84] Remove cr_task/gi_task --- Include/cpython/genobject.h | 2 - Include/internal/pycore_debug_offsets.h | 2 - Include/internal/pycore_genobject.h | 7 - Include/internal/pycore_tstate.h | 1 + Lib/test/test_sys.py | 2 +- Modules/_asynciomodule.c | 32 ++-- Modules/_testexternalinspection.c | 244 +++++++++++++++++------- Objects/genobject.c | 15 -- Python/pystate.c | 2 + 9 files changed, 204 insertions(+), 103 deletions(-) diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index f0c36081d120cc3..f75884e597e2c24 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -32,8 +32,6 @@ PyAPI_DATA(PyTypeObject) PyCoro_Type; PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, PyObject *name, PyObject *qualname); -PyAPI_FUNC(void) _PyCoro_SetTask(PyObject *coro, PyObject *task); - /* --- Asynchronous Generators -------------------------------------------- */ diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 0afa2ce6349a145..38d0cd57ab2c9b9 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -203,7 +203,6 @@ typedef struct _Py_DebugOffsets { uint64_t size; uint64_t gi_name; uint64_t gi_iframe; - uint64_t gi_task; uint64_t gi_frame_state; } gen_object; } _Py_DebugOffsets; @@ -324,7 +323,6 @@ typedef struct _Py_DebugOffsets { .size = sizeof(PyGenObject), \ .gi_name = offsetof(PyGenObject, gi_name), \ .gi_iframe = offsetof(PyGenObject, gi_iframe), \ - .gi_task = offsetof(PyGenObject, gi_task), \ .gi_frame_state = offsetof(PyGenObject, gi_frame_state), \ }, \ } diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index 40ef9098124753c..f6d7e6d367177b6 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -22,13 +22,6 @@ extern "C" { PyObject *prefix##_qualname; \ _PyErr_StackItem prefix##_exc_state; \ PyObject *prefix##_origin_or_finalizer; \ - /* A *borrowed* reference to a task that drives the coroutine. \ - The field is meant to be used by profilers and debuggers only. \ - The main invariant is that a task can't get GC'ed while \ - the coroutine it drives is alive and vice versa. \ - Profilers can use this field to reconstruct the full async \ - call stack of program. */ \ - PyObject *prefix##_task; \ char prefix##_hooks_inited; \ char prefix##_closed; \ char prefix##_running_async; \ diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index a72ef4493b77ca0..d2a171c07c21110 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -22,6 +22,7 @@ typedef struct _PyThreadStateImpl { PyThreadState base; PyObject *asyncio_running_loop; // Strong reference + PyObject *asyncio_running_task; // Strong reference struct _qsbr_thread_state *qsbr; // only used by free-threaded build struct llist_node mem_free_queue; // delayed free queue diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 49767964a01977d..9689ef8e96e072e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1617,7 +1617,7 @@ def bar(cls): check(bar, size('PP')) # generator def get_gen(): yield 1 - check(get_gen(), size('7P4c' + INTERPRETER_FRAME + 'P')) + check(get_gen(), size('6P4c' + INTERPRETER_FRAME + 'P')) # iterator check(iter('abc'), size('lP')) # callable-iterator diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index a0eca5c181cf90f..dbf20c60c470065 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -75,7 +75,6 @@ typedef struct { PyObject *sw_arg; } TaskStepMethWrapper; - #define Future_CheckExact(state, obj) Py_IS_TYPE(obj, state->FutureType) #define Task_CheckExact(state, obj) Py_IS_TYPE(obj, state->TaskType) @@ -113,6 +112,11 @@ typedef struct _Py_AsyncioModuleDebugOffsets { uint64_t task_awaited_by_is_set; uint64_t task_coro; } asyncio_task_object; + struct _asyncio_thread_state { + uint64_t size; + uint64_t asyncio_running_loop; + uint64_t asyncio_running_task; + } asyncio_thread_state; } Py_AsyncioModuleDebugOffsets; GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets AsyncioDebug) @@ -123,6 +127,11 @@ GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets AsyncioDebug) .task_is_task = offsetof(TaskObj, task_is_task), .task_awaited_by_is_set = offsetof(TaskObj, task_awaited_by_is_set), .task_coro = offsetof(TaskObj, task_coro), + }, + .asyncio_thread_state = { + .size = sizeof(_PyThreadStateImpl), + .asyncio_running_loop = offsetof(_PyThreadStateImpl, asyncio_running_loop), + .asyncio_running_task = offsetof(_PyThreadStateImpl, asyncio_running_task), }}; /* State of the _asyncio module */ @@ -219,7 +228,6 @@ typedef struct { TaskObj tail; TaskObj *head; } asyncio_tasks; - } asyncio_state; static inline asyncio_state * @@ -268,9 +276,6 @@ task_step_handle_result_impl(asyncio_state *state, TaskObj *task, PyObject *resu static void clear_task_coro(TaskObj *task) { - if (task->task_coro != NULL && PyCoro_CheckExact(task->task_coro)) { - _PyCoro_SetTask(task->task_coro, NULL); - } Py_CLEAR(task->task_coro); } @@ -279,9 +284,6 @@ static void set_task_coro(TaskObj *task, PyObject *coro) { assert(coro != NULL); - if (PyCoro_CheckExact(coro)) { - _PyCoro_SetTask(coro, (PyObject *)task); - } Py_INCREF(coro); Py_XSETREF(task->task_coro, coro); } @@ -2160,7 +2162,10 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) Py_DECREF(item); return -1; } - Py_DECREF(item); + + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + assert(ts->asyncio_running_task == NULL); + ts->asyncio_running_task = item; // strong ref return 0; } @@ -2185,7 +2190,6 @@ leave_task_predicate(PyObject *item, void *task) static int leave_task(asyncio_state *state, PyObject *loop, PyObject *task) -/*[clinic end generated code: output=0ebf6db4b858fb41 input=51296a46313d1ad8]*/ { int res = _PyDict_DelItemIf(state->current_tasks, loop, leave_task_predicate, task); @@ -2193,6 +2197,9 @@ leave_task(asyncio_state *state, PyObject *loop, PyObject *task) // task was not found return err_leave_task(Py_None, task); } + + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + Py_CLEAR(ts->asyncio_running_task); return res; } @@ -3960,7 +3967,9 @@ module_clear(PyObject *mod) Py_CLEAR(state->iscoroutine_typecache); Py_CLEAR(state->context_kwname); - + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + Py_CLEAR(ts->asyncio_running_loop); + Py_CLEAR(ts->asyncio_running_task); return 0; } @@ -3990,7 +3999,6 @@ module_init(asyncio_state *state) goto fail; } - state->context_kwname = Py_BuildValue("(s)", "context"); if (state->context_kwname == NULL) { goto fail; diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 8f4bb2f59b2f726..ebe92e64d9a22a0 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -60,14 +60,19 @@ #endif struct _Py_AsyncioModuleDebugOffsets { - struct _asyncio_task_object { - uint64_t size; - uint64_t task_name; - uint64_t task_awaited_by; - uint64_t task_is_task; - uint64_t task_awaited_by_is_set; - uint64_t task_coro; - } asyncio_task_object; + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; + struct _asyncio_thread_state { + uint64_t size; + uint64_t asyncio_running_loop; + uint64_t asyncio_running_task; + } asyncio_thread_state; }; #if defined(__APPLE__) && TARGET_OS_OSX @@ -1145,13 +1150,11 @@ parse_async_frame_object( PyObject* result, struct _Py_DebugOffsets* offsets, uintptr_t address, - uintptr_t* task, - uintptr_t* previous_frame + uintptr_t* previous_frame, + uintptr_t* code_object ) { int err; - *task = (uintptr_t)NULL; - ssize_t bytes_read = read_memory( pid, address + offsets->interpreter_frame.previous, @@ -1170,35 +1173,34 @@ parse_async_frame_object( } if (owner == FRAME_OWNED_BY_CSTACK) { - return 0; + return 0; // C frame } - if (owner == FRAME_OWNED_BY_GENERATOR) { - err = read_py_ptr( - pid, - address - offsets->gen_object.gi_iframe + offsets->gen_object.gi_task, - task); - if (err) { - return -1; - } + if (owner != FRAME_OWNED_BY_GENERATOR + && owner != FRAME_OWNED_BY_THREAD) { + PyErr_Format(PyExc_RuntimeError, "Unhandled frame owner %d.\n", owner); + return -1; } - uintptr_t address_of_code_object; err = read_py_ptr( pid, address + offsets->interpreter_frame.executable, - &address_of_code_object + code_object ); if (err) { return -1; } - if ((void*)address_of_code_object == NULL) { + if ((void*)*code_object == NULL) { return 0; } - return parse_code_object( - pid, result, offsets, address_of_code_object, previous_frame); + if (parse_code_object( + pid, result, offsets, *code_object, previous_frame)) { + return -1; + } + + return 1; } static int @@ -1294,6 +1296,77 @@ find_running_frame( return 0; } +static int +find_running_task( + int pid, + uintptr_t runtime_start_address, + _Py_DebugOffsets *local_debug_offsets, + struct _Py_AsyncioModuleDebugOffsets *async_offsets, + uintptr_t *running_task_addr +) { + *running_task_addr = (uintptr_t)NULL; + + off_t interpreter_state_list_head = + local_debug_offsets->runtime_state.interpreters_head; + + uintptr_t address_of_interpreter_state; + int bytes_read = read_memory( + pid, + runtime_start_address + interpreter_state_list_head, + sizeof(void*), + &address_of_interpreter_state); + if (bytes_read == -1) { + return -1; + } + + if (address_of_interpreter_state == 0) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + return -1; + } + + uintptr_t address_of_thread; + bytes_read = read_memory( + pid, + address_of_interpreter_state + + local_debug_offsets->interpreter_state.threads_head, + sizeof(void*), + &address_of_thread); + if (bytes_read == -1) { + return -1; + } + + uintptr_t address_of_running_loop; + // No Python frames are available for us (can happen at tear-down). + if ((void*)address_of_thread == NULL) { + return 0; + } + + bytes_read = read_py_ptr( + pid, + address_of_thread + + async_offsets->asyncio_thread_state.asyncio_running_loop, + &address_of_running_loop); + if (bytes_read == -1) { + return -1; + } + + // no asyncio loop is now running + if ((void*)address_of_running_loop == NULL) { + return 0; + } + + int err = read_ptr( + pid, + address_of_thread + + async_offsets->asyncio_thread_state.asyncio_running_task, + running_task_addr); + if (err) { + return -1; + } + + return 0; +} + static PyObject* get_stack_trace(PyObject* self, PyObject* args) { @@ -1369,14 +1442,6 @@ get_async_stack_trace(PyObject* self, PyObject* args) return NULL; } - uintptr_t address_of_current_frame; - if (find_running_frame( - pid, runtime_start_address, &local_debug_offsets, - &address_of_current_frame) - ) { - return NULL; - } - PyObject* result = PyList_New(1); if (result == NULL) { return NULL; @@ -1391,53 +1456,104 @@ get_async_stack_trace(PyObject* self, PyObject* args) return NULL; } - uintptr_t root_task_addr = (uintptr_t)NULL; + uintptr_t running_task_addr = (uintptr_t)NULL; + if (find_running_task( + pid, runtime_start_address, &local_debug_offsets, &local_async_debug, + &running_task_addr) + ) { + goto result_err; + } + + if ((void*)running_task_addr == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No running task found"); + goto result_err; + } + + uintptr_t running_coro_addr; + if (read_py_ptr( + pid, + running_task_addr + local_async_debug.asyncio_task_object.task_coro, + &running_coro_addr + )) { + goto result_err; + } + + if ((void*)running_coro_addr == NULL) { + PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); + goto result_err; + } + + // note: genobject's gi_iframe is an embedded struct so the address to the offset + // leads directly to its first field: f_executable + uintptr_t address_of_running_task_code_obj; + if (read_py_ptr( + pid, + running_coro_addr + local_debug_offsets.gen_object.gi_iframe, + &address_of_running_task_code_obj + )) { + goto result_err; + } + + if ((void*)address_of_running_task_code_obj == NULL) { + PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); + goto result_err; + } + + uintptr_t address_of_current_frame; + if (find_running_frame( + pid, runtime_start_address, &local_debug_offsets, + &address_of_current_frame) + ) { + goto result_err; + } + + uintptr_t address_of_code_object; while ((void*)address_of_current_frame != NULL) { - int err = parse_async_frame_object( + int res = parse_async_frame_object( pid, calls, &local_debug_offsets, address_of_current_frame, - &root_task_addr, - &address_of_current_frame + &address_of_current_frame, + &address_of_code_object ); - if (err) { + + if (res < 0) { goto result_err; } - if ((void*)root_task_addr != NULL) { + if (address_of_code_object == address_of_running_task_code_obj) { break; } } - if ((void*)root_task_addr != NULL) { - PyObject *tn = parse_task_name( - pid, &local_debug_offsets, &local_async_debug, root_task_addr); - if (tn == NULL) { - goto result_err; - } - if (PyList_Append(result, tn)) { - Py_DECREF(tn); - goto result_err; - } + PyObject *tn = parse_task_name( + pid, &local_debug_offsets, &local_async_debug, running_task_addr); + if (tn == NULL) { + goto result_err; + } + if (PyList_Append(result, tn)) { Py_DECREF(tn); + goto result_err; + } + Py_DECREF(tn); - PyObject* awaited_by = PyList_New(0); - if (awaited_by == NULL) { - goto result_err; - } - if (PyList_Append(result, awaited_by)) { - Py_DECREF(awaited_by); - goto result_err; - } - - if (parse_task_awaited_by( - pid, &local_debug_offsets, &local_async_debug, root_task_addr, awaited_by) - ) { - goto result_err; - } + PyObject* awaited_by = PyList_New(0); + if (awaited_by == NULL) { + goto result_err; + } + if (PyList_Append(result, awaited_by)) { + Py_DECREF(awaited_by); + goto result_err; } + Py_DECREF(awaited_by); + + if (parse_task_awaited_by( + pid, &local_debug_offsets, &local_async_debug, running_task_addr, awaited_by) + ) { + goto result_err; + } return result; diff --git a/Objects/genobject.c b/Objects/genobject.c index 51786cb47fd88dd..49d902ea954d794 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -139,10 +139,6 @@ gen_dealloc(PyObject *self) { PyGenObject *gen = _PyGen_CAST(self); - /* A borrowed reference used only by coroutines and async - frameworks. Just set it to NULL. */ - gen->gi_task = NULL; - _PyObject_GC_UNTRACK(gen); if (gen->gi_weakreflist != NULL) @@ -918,7 +914,6 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) gen->gi_name = Py_NewRef(func->func_name); assert(func->func_qualname != NULL); gen->gi_qualname = Py_NewRef(func->func_qualname); - gen->gi_task = NULL; _PyObject_GC_TRACK(gen); return (PyObject *)gen; } @@ -995,7 +990,6 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, frame->owner = FRAME_OWNED_BY_GENERATOR; assert(PyObject_GC_IsTracked((PyObject *)f)); Py_DECREF(f); - gen->gi_task = NULL; gen->gi_weakreflist = NULL; gen->gi_exc_state.exc_value = NULL; gen->gi_exc_state.previous_item = NULL; @@ -1011,13 +1005,6 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, return (PyObject *)gen; } -void -_PyCoro_SetTask(PyObject *coro, PyObject *task) -{ - assert(PyCoro_CheckExact(coro)); - ((PyCoroObject *)coro)->cr_task = task; -} - PyObject * PyGen_NewWithQualName(PyFrameObject *f, PyObject *name, PyObject *qualname) { @@ -1403,8 +1390,6 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname) return NULL; } - ((PyCoroObject *)coro)->cr_task = NULL; - PyThreadState *tstate = _PyThreadState_GET(); int origin_depth = tstate->coroutine_origin_tracking_depth; diff --git a/Python/pystate.c b/Python/pystate.c index 7df872cd6d7d8a9..59b684a361fb200 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1468,6 +1468,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->dict_global_version = 0; _tstate->asyncio_running_loop = NULL; + _tstate->asyncio_running_task = NULL; tstate->delete_later = NULL; @@ -1667,6 +1668,7 @@ PyThreadState_Clear(PyThreadState *tstate) Py_CLEAR(tstate->threading_local_sentinel); Py_CLEAR(((_PyThreadStateImpl *)tstate)->asyncio_running_loop); + Py_CLEAR(((_PyThreadStateImpl *)tstate)->asyncio_running_task); Py_CLEAR(tstate->dict); Py_CLEAR(tstate->async_exc); From 9f04911d416276b9cc4e6d4f9f4eb2b220a5c28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 13 Nov 2024 16:56:50 +0100 Subject: [PATCH 50/84] Fix refleaks --- Modules/_testexternalinspection.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index ebe92e64d9a22a0..7481dac58df5d61 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -713,6 +713,7 @@ parse_coro_chain( if (PyList_Append(render_to, name)) { return -1; } + Py_DECREF(name); int gi_frame_state; err = read_int( @@ -886,6 +887,7 @@ parse_task( if (PyList_Append(render_to, result)) { goto err; } + Py_DECREF(result); PyObject *awaited_by = PyList_New(0); if (awaited_by == NULL) { @@ -1546,7 +1548,6 @@ get_async_stack_trace(PyObject* self, PyObject* args) Py_DECREF(awaited_by); goto result_err; } - Py_DECREF(awaited_by); if (parse_task_awaited_by( From 07748059f4ba983b3f65f2e4855c2461813f5566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 13 Nov 2024 17:17:31 +0100 Subject: [PATCH 51/84] Rename "asyncio.stack" to "asyncio.graph" --- .../{asyncio-stack.rst => asyncio-graph.rst} | 6 ++-- Doc/library/asyncio.rst | 2 +- Lib/asyncio/__init__.py | 4 +-- Lib/asyncio/{stack.py => graph.py} | 28 +++++++++---------- .../{test_stack.py => test_graph.py} | 0 5 files changed, 20 insertions(+), 20 deletions(-) rename Doc/library/{asyncio-stack.rst => asyncio-graph.rst} (96%) rename Lib/asyncio/{stack.py => graph.py} (90%) rename Lib/test/test_asyncio/{test_stack.py => test_graph.py} (100%) diff --git a/Doc/library/asyncio-stack.rst b/Doc/library/asyncio-graph.rst similarity index 96% rename from Doc/library/asyncio-stack.rst rename to Doc/library/asyncio-graph.rst index fab72521bb9df8e..55f431cb200f854 100644 --- a/Doc/library/asyncio-stack.rst +++ b/Doc/library/asyncio-graph.rst @@ -1,17 +1,17 @@ .. currentmodule:: asyncio -.. _asyncio-stack: +.. _asyncio-graph: =================== Stack Introspection =================== -**Source code:** :source:`Lib/asyncio/stack.py` +**Source code:** :source:`Lib/asyncio/graph.py` ------------------------------------- -asyncio has powerful runtime call stack introspection utilities +asyncio has powerful runtime call graph introspection utilities to trace the entire call graph of a running coroutine or task, or a suspended *future*. These utilities and the underlying machinery can be used by users in their Python code or by external profilers diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 5098805f26cbd51..7d368dae49dc1d7 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -99,7 +99,7 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst - asyncio-stack.rst + asyncio-graph.rst .. toctree:: :caption: Low-level APIs diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index b05c3fdbdf9641c..6576070ed77a2b1 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -14,7 +14,7 @@ from .protocols import * from .runners import * from .queues import * -from .stack import * +from .graph import * from .streams import * from .subprocess import * from .tasks import * @@ -32,7 +32,7 @@ protocols.__all__ + runners.__all__ + queues.__all__ + - stack.__all__ + + graph.__all__ + streams.__all__ + subprocess.__all__ + tasks.__all__ + diff --git a/Lib/asyncio/stack.py b/Lib/asyncio/graph.py similarity index 90% rename from Lib/asyncio/stack.py rename to Lib/asyncio/graph.py index f68e80f55063af0..557e43486348974 100644 --- a/Lib/asyncio/stack.py +++ b/Lib/asyncio/graph.py @@ -1,4 +1,4 @@ -"""Introspection utils for tasks call stacks.""" +"""Introspection utils for tasks call graphs.""" import dataclasses import sys @@ -18,7 +18,7 @@ # Sadly, we can't re-use the traceback's module datastructures as those # are tailored for error reporting, whereas we need to represent an -# async call stack. +# async call graph. # # Going with pretty verbose names as we'd like to export them to the # top level asyncio namespace, and want to avoid future name clashes. @@ -36,7 +36,7 @@ class FutureCallGraph: awaited_by: list[FutureCallGraph] -def _build_stack_for_future(future: futures.Future) -> FutureCallGraph: +def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: if not isinstance(future, futures.Future): raise TypeError( f"{future!r} object does not appear to be compatible " @@ -64,7 +64,7 @@ def _build_stack_for_future(future: futures.Future) -> FutureCallGraph: if future._asyncio_awaited_by: for parent in future._asyncio_awaited_by: - awaited_by.append(_build_stack_for_future(parent)) + awaited_by.append(_build_graph_for_future(parent)) st.reverse() return FutureCallGraph(future, st, awaited_by) @@ -75,9 +75,9 @@ def capture_call_graph( future: futures.Future | None = None, depth: int = 1, ) -> FutureCallGraph | None: - """Capture async call stack for the current task or the provided Future. + """Capture async call graph for the current task or the provided Future. - The stack is represented with three data structures: + The graph is represented with three data structures: * FutureCallGraph(future, call_stack, awaited_by) @@ -109,7 +109,7 @@ def capture_call_graph( # if yes - check if the passed future is the currently # running task or not. if loop is None or future is not tasks.current_task(loop=loop): - return _build_stack_for_future(future) + return _build_graph_for_future(future) # else: future is the current task, move on. else: if loop is None: @@ -151,7 +151,7 @@ def capture_call_graph( awaited_by = [] if future._asyncio_awaited_by: for parent in future._asyncio_awaited_by: - awaited_by.append(_build_stack_for_future(parent)) + awaited_by.append(_build_graph_for_future(parent)) return FutureCallGraph(future, call_stack, awaited_by) @@ -162,7 +162,7 @@ def print_call_graph( file: typing.TextIO | None = None, depth: int = 1, ) -> None: - """Print async call stack for the current task or the provided Future.""" + """Print async call graph for the current task or the provided Future.""" def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None: def add_line(line: str) -> None: @@ -221,16 +221,16 @@ def add_line(line: str) -> None: for fut in st.awaited_by: render_level(fut, buf, level + 1) - stack = capture_call_graph(future=future, depth=depth + 1) - if stack is None: + graph = capture_call_graph(future=future, depth=depth + 1) + if graph is None: return try: buf: list[str] = [] - render_level(stack, buf, 0) + render_level(graph, buf, 0) rendered = '\n'.join(buf) print(rendered, file=file) finally: - # 'stack' has references to frames so we should + # 'graph' has references to frames so we should # make sure it's GC'ed as soon as we don't need it. - del stack + del graph diff --git a/Lib/test/test_asyncio/test_stack.py b/Lib/test/test_asyncio/test_graph.py similarity index 100% rename from Lib/test/test_asyncio/test_stack.py rename to Lib/test/test_asyncio/test_graph.py From 3048493dfeaebd03281b0121c0573a3191e47986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 13 Nov 2024 17:24:23 +0100 Subject: [PATCH 52/84] Allow returning a string with `format_call_graph` --- Doc/library/asyncio-graph.rst | 14 ++------------ Lib/asyncio/graph.py | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 55f431cb200f854..f080ab0e3198aaa 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -63,19 +63,9 @@ and debuggers. | File 'taskgroups.py', line 107, in async TaskGroup.__aexit__() | File 't2.py', line 7, in async main() - For rendering the call stack to a string the following pattern - should be used: - - .. code-block:: python - - import io - - ... - - buf = io.StringIO() - asyncio.print_call_graph(file=buf) - output = buf.getvalue() +.. function:: format_call_graph(*, future=None, depth=1) + Like :func:`print_call_graph`, but returns a string. .. function:: capture_call_graph(*, future=None) diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index 557e43486348974..fb613f034ba2d34 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -11,6 +11,7 @@ __all__ = ( 'capture_call_graph', + 'format_call_graph', 'print_call_graph', 'FrameCallGraphEntry', 'FutureCallGraph', @@ -156,13 +157,15 @@ def capture_call_graph( return FutureCallGraph(future, call_stack, awaited_by) -def print_call_graph( +def format_call_graph( *, future: futures.Future | None = None, - file: typing.TextIO | None = None, depth: int = 1, -) -> None: - """Print async call graph for the current task or the provided Future.""" +) -> str: + """Return async call graph as a string for `future`. + + If `future` is not provided, format the call graph for the current task. + """ def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None: def add_line(line: str) -> None: @@ -228,9 +231,17 @@ def add_line(line: str) -> None: try: buf: list[str] = [] render_level(graph, buf, 0) - rendered = '\n'.join(buf) - print(rendered, file=file) + return '\n'.join(buf) finally: # 'graph' has references to frames so we should # make sure it's GC'ed as soon as we don't need it. del graph + +def print_call_graph( + *, + future: futures.Future | None = None, + file: typing.TextIO | None = None, + depth: int = 1, +) -> None: + """Print async call graph for the current task or the provided Future.""" + print(format_call_graph(future=future, depth=depth), file=file) From 1f42873bcb656f05816d155671e9c7dd84dd3968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 13 Nov 2024 17:34:08 +0100 Subject: [PATCH 53/84] Add test for eager task factory support --- Lib/test/test_external_inspection.py | 81 ++++++++++++++++------------ 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 9ef3ea86dd002f1..720a6f5170c5ed9 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -109,42 +109,55 @@ async def main(): tg.create_task(c1(task), name="sub_main_1") tg.create_task(c1(task), name="sub_main_2") - asyncio.run(main()) + def new_eager_loop(): + loop = asyncio.new_event_loop() + eager_task_factory = asyncio.create_eager_task_factory(asyncio.Task) + loop.set_task_factory(eager_task_factory) + return loop + + asyncio.run(main(), loop_factory={TASK_FACTORY}) """) stack_trace = None - with os_helper.temp_dir() as work_dir: - script_dir = os.path.join(work_dir, "script_pkg") - os.mkdir(script_dir) - fifo = f"{work_dir}/the_fifo" - os.mkfifo(fifo) - script_name = _make_test_script(script_dir, 'script', script) - try: - p = subprocess.Popen([sys.executable, script_name, str(fifo)]) - with open(fifo, "r") as fifo_file: - response = fifo_file.read() - self.assertEqual(response, "ready") - stack_trace = get_async_stack_trace(p.pid) - except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") - finally: - os.remove(fifo) - p.kill() - p.terminate() - p.wait(timeout=SHORT_TIMEOUT) - - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - - expected_stack_trace = [ - ["c5", "c4", "c3", "c2"], - "c2_root", - [ - [["main"], "Task-1", []], - [["c1"], "sub_main_1", [[["main"], "Task-1", []]]], - [["c1"], "sub_main_2", [[["main"], "Task-1", []]]], - ], - ] - self.assertEqual(stack_trace, expected_stack_trace) + for task_factory_variant in "asyncio.new_event_loop", "new_eager_loop": + with ( + self.subTest(task_factory_variant=task_factory_variant), + os_helper.temp_dir() as work_dir, + ): + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + fifo = f"{work_dir}/the_fifo" + os.mkfifo(fifo) + script_name = _make_test_script(script_dir, 'script', script.format(TASK_FACTORY=task_factory_variant)) + try: + p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + with open(fifo, "r") as fifo_file: + response = fifo_file.read() + self.assertEqual(response, "ready") + stack_trace = get_async_stack_trace(p.pid) + except PermissionError: + self.skipTest("Insufficient permissions to read the stack trace") + finally: + os.remove(fifo) + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # sets are unordered, so we want to sort "awaited_by"s + stack_trace[2].sort(key=lambda x: x[1]) + + root_task = "Task-1" + if task_factory_variant == "new_eager_loop": + root_task = "None" + expected_stack_trace = [ + ["c5", "c4", "c3", "c2"], + "c2_root", + [ + [["main"], root_task, []], + [["c1"], "sub_main_1", [[["main"], root_task, []]]], + [["c1"], "sub_main_2", [[["main"], root_task, []]]], + ], + ] + self.assertEqual(stack_trace, expected_stack_trace) @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") From 03ed5c188e9a15a709622909b2c2d75edc586543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 13 Nov 2024 17:59:51 +0100 Subject: [PATCH 54/84] Make _asyncio_awaited_by a frozenset in the Python version as well per Kumar's wishes --- Lib/asyncio/futures.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 8bd47a90c7a83a5..71fd283acfa5633 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -68,7 +68,7 @@ class Future: _asyncio_future_blocking = False # Used by the capture_call_stack() API. - _asyncio_awaited_by = None + __asyncio_awaited_by = None __log_traceback = False @@ -119,6 +119,12 @@ def _log_traceback(self, val): raise ValueError('_log_traceback can only be set to False') self.__log_traceback = False + @property + def _asyncio_awaited_by(self): + if self.__asyncio_awaited_by is None: + return None + return frozenset(self.__asyncio_awaited_by) + def get_loop(self): """Return the event loop the Future is bound to.""" loop = self._loop @@ -442,9 +448,9 @@ def future_add_to_awaited_by(fut, waiter, /): # Note that there's an accelerated version of this function # shadowing this implementation later in this file. if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): - if fut._asyncio_awaited_by is None: - fut._asyncio_awaited_by = set() - fut._asyncio_awaited_by.add(waiter) + if fut._Future__asyncio_awaited_by is None: + fut._Future__asyncio_awaited_by = set() + fut._Future__asyncio_awaited_by.add(waiter) def future_discard_from_awaited_by(fut, waiter, /): @@ -455,8 +461,8 @@ def future_discard_from_awaited_by(fut, waiter, /): # Note that there's an accelerated version of this function # shadowing this implementation later in this file. if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture): - if fut._asyncio_awaited_by is not None: - fut._asyncio_awaited_by.discard(waiter) + if fut._Future__asyncio_awaited_by is not None: + fut._Future__asyncio_awaited_by.discard(waiter) try: From 21f9ea91eb95b856ac05bb3e6189b0d53368e458 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 22 Nov 2024 17:01:59 -0800 Subject: [PATCH 55/84] Address picnixz' feedback --- Doc/library/asyncio-graph.rst | 28 ++++++++++++------------- Doc/library/inspect.rst | 2 +- Include/internal/pycore_debug_offsets.h | 2 +- Lib/asyncio/__init__.py | 4 ++-- Objects/frameobject.c | 3 +-- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index f080ab0e3198aaa..4ed5b2609b8b4d6 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -17,7 +17,7 @@ a suspended *future*. These utilities and the underlying machinery can be used by users in their Python code or by external profilers and debuggers. -.. versionadded:: 3.14 +.. versionadded:: next .. function:: print_call_graph(*, future=None, file=None, depth=1) @@ -44,11 +44,11 @@ and debuggers. import asyncio async def test(): - asyncio.print_call_graph() + asyncio.print_call_graph() async def main(): - async with asyncio.TaskGroup() as g: - g.create_task(test()) + async with asyncio.TaskGroup() as g: + g.create_task(test()) asyncio.run(main()) @@ -77,15 +77,15 @@ and debuggers. current task, the function returns ``None``. If the function is called on *the current task*, the optional - keyword-only ``depth`` argument can be used to skip the specified + keyword-only *depth* argument can be used to skip the specified number of frames from top of the stack. Returns a ``FutureCallGraph`` data class object: * ``FutureCallGraph(future, call_stack, awaited_by)`` - Where 'future' is a reference to a *Future* or a *Task* - (or their subclasses.) + Where *future* is a reference to a :class:`Future` or + a :class:`Task` (or their subclasses.) ``call_stack`` is a list of ``FrameCallGraphEntry`` objects. @@ -93,7 +93,7 @@ and debuggers. * ``FrameCallGraphEntry(frame)`` - Where ``frame`` is a frame object of a regular Python function + Where *frame* is a frame object of a regular Python function in the call stack. @@ -102,7 +102,7 @@ Low level utility functions To introspect an async call graph asyncio requires cooperation from control flow structures, such as :func:`shield` or :class:`TaskGroup`. -Any time an intermediate ``Future`` object with low-level APIs like +Any time an intermediate :class:`Future` object with low-level APIs like :meth:`Future.add_done_callback() ` is involved, the following two functions should be used to inform *asyncio* about how exactly such intermediate future objects are connected with @@ -114,11 +114,11 @@ the tasks they wrap or control. Record that *future* is awaited on by *waiter*. Both *future* and *waiter* must be instances of - :class:`asyncio.Future ` or :class:`asyncio.Task ` or - their subclasses, otherwise the call would have no effect. + :class:`Future` or :class:`Task` or their subclasses, + otherwise the call would have no effect. A call to ``future_add_to_awaited_by()`` must be followed by an - eventual call to the ``future_discard_from_awaited_by()`` function + eventual call to the :func:`future_discard_from_awaited_by` function with the same arguments. @@ -127,5 +127,5 @@ the tasks they wrap or control. Record that *future* is no longer awaited on by *waiter*. Both *future* and *waiter* must be instances of - :class:`asyncio.Future ` or :class:`asyncio.Task ` or - their subclasses, otherwise the call would have no effect. + :class:`Future` or :class:`Task` or their subclasses, otherwise + the call would have no effect. diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 91c740f433decc1..0902d64f9bd22a5 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -316,7 +316,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Add ``__builtins__`` attribute to functions. -.. versionchanged:: 3.14 +.. versionchanged:: next Add ``f_generator`` attribute to frames. diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 38d0cd57ab2c9b9..34debf35d14df4d 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -16,7 +16,7 @@ extern "C" { #endif // Macros to burn global values in custom sections so out-of-process -// profilers can locate them easily +// profilers can locate them easily. #define GENERATE_DEBUG_SECTION(name, declaration) \ _GENERATE_DEBUG_SECTION_WINDOWS(name) \ diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 6576070ed77a2b1..2432e2dad74c23d 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -10,11 +10,11 @@ from .events import * from .exceptions import * from .futures import * +from .graph import * from .locks import * from .protocols import * from .runners import * from .queues import * -from .graph import * from .streams import * from .subprocess import * from .tasks import * @@ -28,11 +28,11 @@ events.__all__ + exceptions.__all__ + futures.__all__ + + graph.__all__ + locks.__all__ + protocols.__all__ + runners.__all__ + queues.__all__ + - graph.__all__ + streams.__all__ + subprocess.__all__ + tasks.__all__ + diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 92f3aea5d45472b..da6a9b453c7028e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1683,8 +1683,7 @@ static PyObject * frame_getgenerator(PyFrameObject *f, void *arg) { if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { PyObject *gen = (PyObject *)_PyGen_GetGeneratorFromFrame(f->f_frame); - Py_INCREF(gen); - return gen; + return Py_NewRef(gen); } Py_RETURN_NONE; } From 8a43dfa0b6e84649fd4a9ca1d9d3b895a4ac4ce2 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 22 Nov 2024 20:22:03 -0800 Subject: [PATCH 56/84] Blacken the C/Py test code by hand, by request from picnixz --- Lib/test/test_external_inspection.py | 58 +++++++--- Modules/_testexternalinspection.c | 159 ++++++++++++++++++--------- 2 files changed, 148 insertions(+), 69 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 720a6f5170c5ed9..1c54599de6ef279 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -15,7 +15,8 @@ from _testexternalinspection import get_stack_trace from _testexternalinspection import get_async_stack_trace except ImportError: - raise unittest.SkipTest("Test only runs when _testexternalinspection is available") + raise unittest.SkipTest( + "Test only runs when _testexternalinspection is available") def _make_test_script(script_dir, script_basename, source): to_return = make_script(script_dir, script_basename, source) @@ -24,8 +25,10 @@ def _make_test_script(script_dir, script_basename, source): class TestGetStackTrace(unittest.TestCase): - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_remote_stack_trace(self): # Spawn a process with some realistic Python code script = textwrap.dedent("""\ @@ -75,8 +78,10 @@ def foo(): ] self.assertEqual(stack_trace, expected_stack_trace) - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_async_remote_stack_trace(self): # Spawn a process with some realistic Python code script = textwrap.dedent("""\ @@ -111,7 +116,8 @@ async def main(): def new_eager_loop(): loop = asyncio.new_event_loop() - eager_task_factory = asyncio.create_eager_task_factory(asyncio.Task) + eager_task_factory = asyncio.create_eager_task_factory( + asyncio.Task) loop.set_task_factory(eager_task_factory) return loop @@ -127,15 +133,20 @@ def new_eager_loop(): os.mkdir(script_dir) fifo = f"{work_dir}/the_fifo" os.mkfifo(fifo) - script_name = _make_test_script(script_dir, 'script', script.format(TASK_FACTORY=task_factory_variant)) + script_name = _make_test_script( + script_dir, 'script', + script.format(TASK_FACTORY=task_factory_variant)) try: - p = subprocess.Popen([sys.executable, script_name, str(fifo)]) + p = subprocess.Popen( + [sys.executable, script_name, str(fifo)] + ) with open(fifo, "r") as fifo_file: response = fifo_file.read() self.assertEqual(response, "ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace") finally: os.remove(fifo) p.kill() @@ -159,8 +170,10 @@ def new_eager_loop(): ] self.assertEqual(stack_trace, expected_stack_trace) - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_asyncgen_remote_stack_trace(self): # Spawn a process with some realistic Python code script = textwrap.dedent("""\ @@ -210,11 +223,15 @@ async def main(): # sets are unordered, so we want to sort "awaited_by"s stack_trace[2].sort(key=lambda x: x[1]) - expected_stack_trace = [['gen_nested_call', 'gen', 'main'], 'Task-1', []] + expected_stack_trace = [ + ['gen_nested_call', 'gen', 'main'], 'Task-1', [] + ] self.assertEqual(stack_trace, expected_stack_trace) - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_async_gather_remote_stack_trace(self): # Spawn a process with some realistic Python code script = textwrap.dedent("""\ @@ -255,7 +272,8 @@ async def main(): self.assertEqual(response, "ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace") finally: os.remove(fifo) p.kill() @@ -265,11 +283,15 @@ async def main(): # sets are unordered, so we want to sort "awaited_by"s stack_trace[2].sort(key=lambda x: x[1]) - expected_stack_trace = [['deep', 'c1'], 'Task-2', [[['main'], 'Task-1', []]]] + expected_stack_trace = [ + ['deep', 'c1'], 'Task-2', [[['main'], 'Task-1', []]] + ] self.assertEqual(stack_trace, expected_stack_trace) - @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", "Test only runs on Linux and MacOS") - @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") + @unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux", + "Test only runs on Linux and MacOS") + @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support") def test_self_trace(self): stack_trace = get_stack_trace(os.getpid()) self.assertEqual(stack_trace[0], "test_self_trace") diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 7481dac58df5d61..82b0f95a416b3cc 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -77,8 +77,12 @@ struct _Py_AsyncioModuleDebugOffsets { #if defined(__APPLE__) && TARGET_OS_OSX static uintptr_t -return_section_address(const char* section, mach_port_t proc_ref, uintptr_t base, void* map) -{ +return_section_address( + const char* section, + mach_port_t proc_ref, + uintptr_t base, + void* map +) { struct mach_header_64* hdr = (struct mach_header_64*)map; int ncmds = hdr->ncmds; @@ -88,7 +92,7 @@ return_section_address(const char* section, mach_port_t proc_ref, uintptr_t base mach_vm_size_t size = 0; mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); mach_vm_address_t address = (mach_vm_address_t)base; - vm_region_basic_info_data_64_t region_info; + vm_region_basic_info_data_64_t r_info; mach_port_t object_name; uintptr_t vmaddr = 0; @@ -99,24 +103,26 @@ return_section_address(const char* section, mach_port_t proc_ref, uintptr_t base if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { while (cmd->filesize != size) { address += size; - if (mach_vm_region( - proc_ref, - &address, - &size, - VM_REGION_BASIC_INFO_64, - (vm_region_info_t)®ion_info, // cppcheck-suppress [uninitvar] - &count, - &object_name) - != KERN_SUCCESS) - { - PyErr_SetString(PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + kern_return_t ret = mach_vm_region( + proc_ref, + &address, + &size, + VM_REGION_BASIC_INFO_64, + (vm_region_info_t)&r_info, // cppcheck-suppress [uninitvar] + &count, + &object_name + ); + if (ret != KERN_SUCCESS) { + PyErr_SetString( + PyExc_RuntimeError, "Cannot get any more VM maps.\n"); return 0; } } int nsects = cmd->nsects; - struct section_64* sec = - (struct section_64*)((void*)cmd + sizeof(struct segment_command_64)); + struct section_64* sec = (struct section_64*)( + (void*)cmd + sizeof(struct segment_command_64) + ); for (int j = 0; j < nsects; j++) { if (strcmp(sec[j].sectname, section) == 0) { return base + sec[j].addr - vmaddr; @@ -131,8 +137,13 @@ return_section_address(const char* section, mach_port_t proc_ref, uintptr_t base } static uintptr_t -search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_size_t size, mach_port_t proc_ref) -{ +search_section_in_file( + const char* secname, + char* path, + uintptr_t base, + mach_vm_size_t size, + mach_port_t proc_ref +) { int fd = open(path, O_RDONLY); if (fd == -1) { PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); @@ -141,7 +152,8 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ struct stat fs; if (fstat(fd, &fs) == -1) { - PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); + PyErr_Format( + PyExc_RuntimeError, "Cannot get size of binary %s\n", path); close(fd); return 0; } @@ -161,7 +173,9 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ case MH_CIGAM: case FAT_MAGIC: case FAT_CIGAM: - PyErr_SetString(PyExc_RuntimeError, "32-bit Mach-O binaries are not supported"); + PyErr_SetString( + PyExc_RuntimeError, + "32-bit Mach-O binaries are not supported"); break; case MH_MAGIC_64: case MH_CIGAM_64: @@ -216,10 +230,10 @@ search_map_for_section(pid_t pid, const char* secname, const char* substr) { VM_REGION_BASIC_INFO_64, (vm_region_info_t)®ion_info, &count, - &object_name) - == KERN_SUCCESS) + &object_name) == KERN_SUCCESS) { - int path_len = proc_regionfilename(pid, address, map_filename, MAXPATHLEN); + int path_len = proc_regionfilename( + pid, address, map_filename, MAXPATHLEN); if (path_len == 0) { address += size; continue; @@ -240,7 +254,8 @@ search_map_for_section(pid_t pid, const char* secname, const char* substr) { if (!match_found && strncmp(filename, substr, strlen(substr)) == 0) { match_found = 1; - return search_section_in_file(secname, map_filename, address, size, proc_ref); + return search_section_in_file( + secname, map_filename, address, size, proc_ref); } address += size; @@ -270,7 +285,10 @@ find_map_start_address(pid_t pid, char* result_filename, const char* map) uintptr_t result_address = 0; while (fgets(line, sizeof(line), maps_file) != NULL) { unsigned long start_address = 0; - sscanf(line, "%lx-%*x %*s %*s %*s %*s %s", &start_address, map_filename); + sscanf( + line, "%lx-%*x %*s %*s %*s %*s %s", + &start_address, map_filename + ); char* filename = strrchr(map_filename, '/'); if (filename != NULL) { filename++; // Move past the '/' @@ -328,7 +346,8 @@ search_map_for_section(pid_t pid, const char* secname, const char* map) Elf_Ehdr* elf_header = (Elf_Ehdr*)file_memory; - Elf_Shdr* section_header_table = (Elf_Shdr*)(file_memory + elf_header->e_shoff); + Elf_Shdr* section_header_table = + (Elf_Shdr*)(file_memory + elf_header->e_shoff); Elf_Shdr* shstrtab_section = §ion_header_table[elf_header->e_shstrndx]; char* shstrtab = (char*)(file_memory + shstrtab_section->sh_offset); @@ -344,7 +363,9 @@ search_map_for_section(pid_t pid, const char* secname, const char* map) } } - Elf_Phdr* program_header_table = (Elf_Phdr*)(file_memory + elf_header->e_phoff); + Elf_Phdr* program_header_table = + (Elf_Phdr*)(file_memory + elf_header->e_phoff); + // Find the first PT_LOAD segment Elf_Phdr* first_load_segment = NULL; for (int i = 0; i < elf_header->e_phnum; i++) { @@ -355,8 +376,10 @@ search_map_for_section(pid_t pid, const char* secname, const char* map) } if (section != NULL && first_load_segment != NULL) { - uintptr_t elf_load_addr = first_load_segment->p_vaddr - - (first_load_segment->p_vaddr % first_load_segment->p_align); + uintptr_t elf_load_addr = + first_load_segment->p_vaddr - ( + first_load_segment->p_vaddr % first_load_segment->p_align + ); result = start_address + (uintptr_t)section->sh_addr - elf_load_addr; } @@ -426,13 +449,19 @@ read_memory(pid_t pid, uintptr_t remote_address, size_t len, void* dst) if (kr != KERN_SUCCESS) { switch (kr) { case KERN_PROTECTION_FAILURE: - PyErr_SetString(PyExc_PermissionError, "Not enough permissions to read memory"); + PyErr_SetString( + PyExc_PermissionError, + "Not enough permissions to read memory"); break; case KERN_INVALID_ARGUMENT: - PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_read_overwrite"); + PyErr_SetString( + PyExc_PermissionError, + "Invalid argument to mach_vm_read_overwrite"); break; default: - PyErr_SetString(PyExc_RuntimeError, "Unknown error reading memory"); + PyErr_SetString( + PyExc_RuntimeError, + "Unknown error reading memory"); } return -1; } @@ -444,8 +473,13 @@ read_memory(pid_t pid, uintptr_t remote_address, size_t len, void* dst) } static int -read_string(pid_t pid, _Py_DebugOffsets* debug_offsets, uintptr_t address, char* buffer, Py_ssize_t size) -{ +read_string( + pid_t pid, + _Py_DebugOffsets* debug_offsets, + uintptr_t address, + char* buffer, + Py_ssize_t size +) { Py_ssize_t len; ssize_t bytes_read = read_memory( pid, @@ -565,18 +599,21 @@ read_py_str( } static long -read_py_long( - pid_t pid, - _Py_DebugOffsets* offsets, - uintptr_t address) { +read_py_long(pid_t pid, _Py_DebugOffsets* offsets, uintptr_t address) +{ unsigned int shift = PYLONG_BITS_IN_DIGIT; ssize_t size; uintptr_t lv_tag; - int bytes_read = read_memory(pid, address + offsets->long_object.lv_tag, sizeof(uintptr_t), &lv_tag); + + int bytes_read = read_memory( + pid, address + offsets->long_object.lv_tag, + sizeof(uintptr_t), + &lv_tag); if (bytes_read == -1) { return -1; } + int negative = (lv_tag & 3) == 2; size = lv_tag >> 3; @@ -586,9 +623,16 @@ read_py_long( char *digits = (char *)PyMem_RawMalloc(size * sizeof(digit)); if (!digits) { + PyErr_NoMemory(); return -1; } - bytes_read = read_memory(pid, address + offsets->long_object.ob_digit, sizeof(digit) * size, digits); + + bytes_read = read_memory( + pid, + address + offsets->long_object.ob_digit, + sizeof(digit) * size, + digits + ); if (bytes_read < 0) { goto error; } @@ -597,7 +641,9 @@ read_py_long( for (ssize_t i = 0; i < size; ++i) { long long factor; - if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), &factor)) { + if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), + &factor) + ) { goto error; } if (__builtin_add_overflow(value, factor, &value)) { @@ -847,7 +893,8 @@ parse_task( Py_DECREF(call_stack); if (is_task) { - PyObject *tn = parse_task_name(pid, offsets, async_offsets, task_address); + PyObject *tn = parse_task_name( + pid, offsets, async_offsets, task_address); if (tn == NULL) { goto err; } @@ -900,7 +947,9 @@ parse_task( /* we can operate on a borrowed one to simplify cleanup */ Py_DECREF(awaited_by); - if (parse_task_awaited_by(pid, offsets, async_offsets, task_address, awaited_by)) { + if (parse_task_awaited_by(pid, offsets, async_offsets, + task_address, awaited_by) + ) { goto err; } @@ -1372,8 +1421,11 @@ find_running_task( static PyObject* get_stack_trace(PyObject* self, PyObject* args) { -#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); +#if (!defined(__linux__) && !defined(__APPLE__)) || \ + (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString( + PyExc_RuntimeError, + "get_stack_trace is not supported on this platform"); return NULL; #endif int pid; @@ -1422,8 +1474,11 @@ get_stack_trace(PyObject* self, PyObject* args) static PyObject* get_async_stack_trace(PyObject* self, PyObject* args) { -#if (!defined(__linux__) && !defined(__APPLE__)) || (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString(PyExc_RuntimeError, "get_stack_trace is not supported on this platform"); +#if (!defined(__linux__) && !defined(__APPLE__)) || \ + (defined(__linux__) && !HAVE_PROCESS_VM_READV) + PyErr_SetString( + PyExc_RuntimeError, + "get_stack_trace is not supported on this platform"); return NULL; #endif int pid; @@ -1485,8 +1540,8 @@ get_async_stack_trace(PyObject* self, PyObject* args) goto result_err; } - // note: genobject's gi_iframe is an embedded struct so the address to the offset - // leads directly to its first field: f_executable + // note: genobject's gi_iframe is an embedded struct so the address to + // the offset leads directly to its first field: f_executable uintptr_t address_of_running_task_code_obj; if (read_py_ptr( pid, @@ -1551,7 +1606,8 @@ get_async_stack_trace(PyObject* self, PyObject* args) Py_DECREF(awaited_by); if (parse_task_awaited_by( - pid, &local_debug_offsets, &local_async_debug, running_task_addr, awaited_by) + pid, &local_debug_offsets, &local_async_debug, + running_task_addr, awaited_by) ) { goto result_err; } @@ -1589,7 +1645,8 @@ PyInit__testexternalinspection(void) #ifdef Py_GIL_DISABLED PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); #endif - int rc = PyModule_AddIntConstant(mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); + int rc = PyModule_AddIntConstant( + mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); if (rc < 0) { Py_DECREF(mod); return NULL; From b3fae687af7a594f1166ea5892d5ec4ab495f7dd Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 22 Nov 2024 20:33:21 -0800 Subject: [PATCH 57/84] More style fixes per picnixz' suggestions --- Modules/_testexternalinspection.c | 47 +++++++++++++++++-------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 82b0f95a416b3cc..d0fb3ca2736c6d8 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -98,7 +98,7 @@ return_section_address( for (int i = 0; cmd_cnt < 2 && i < ncmds; i++) { if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__TEXT") == 0) { - vmaddr = cmd->vmaddr; + vmaddr = cmd->vmaddr; } if (cmd->cmd == LC_SEGMENT_64 && strcmp(cmd->segname, "__DATA") == 0) { while (cmd->filesize != size) { @@ -354,8 +354,11 @@ search_map_for_section(pid_t pid, const char* secname, const char* map) Elf_Shdr* section = NULL; for (int i = 0; i < elf_header->e_shnum; i++) { - char* this_sec_name = shstrtab + section_header_table[i].sh_name; - // Move 1 character to account for the leading "." + const char* this_sec_name = ( + shstrtab + + section_header_table[i].sh_name + + 1 // "+1" accounts for the leading "." + ); this_sec_name += 1; if (strcmp(secname, this_sec_name) == 0) { section = §ion_header_table[i]; @@ -487,7 +490,7 @@ read_string( sizeof(Py_ssize_t), &len ); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } if (len >= size) { @@ -496,7 +499,7 @@ read_string( } size_t offset = debug_offsets->unicode_object.asciiobject_size; bytes_read = read_memory(pid, address + offset, len, buffer); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } buffer[len] = '\0'; @@ -508,7 +511,7 @@ static inline int read_ptr(pid_t pid, uintptr_t address, uintptr_t *ptr_addr) { int bytes_read = read_memory(pid, address, sizeof(void*), ptr_addr); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } return 0; @@ -518,7 +521,7 @@ static inline int read_ssize_t(pid_t pid, uintptr_t address, Py_ssize_t *size) { int bytes_read = read_memory(pid, address, sizeof(Py_ssize_t), size); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } return 0; @@ -610,7 +613,7 @@ read_py_long(pid_t pid, _Py_DebugOffsets* offsets, uintptr_t address) pid, address + offsets->long_object.lv_tag, sizeof(uintptr_t), &lv_tag); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -641,7 +644,7 @@ read_py_long(pid_t pid, _Py_DebugOffsets* offsets, uintptr_t address) for (ssize_t i = 0; i < size; ++i) { long long factor; - if (__builtin_mul_overflow(digits[i], (1Lu << (ssize_t)(shift * i)), + if (__builtin_mul_overflow(digits[i], (1UL << (ssize_t)(shift * i)), &factor) ) { goto error; @@ -652,7 +655,7 @@ read_py_long(pid_t pid, _Py_DebugOffsets* offsets, uintptr_t address) } PyMem_RawFree(digits); if (negative) { - value = -1 * value; + value *= -1; } return value; error: @@ -883,7 +886,7 @@ parse_task( PyObject *call_stack = PyList_New(0); if (call_stack == NULL) { - return -1; + goto err; } if (PyList_Append(result, call_stack)) { Py_DECREF(call_stack); @@ -1124,7 +1127,7 @@ parse_code_object( sizeof(void*), &address_of_function_name ); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1164,7 +1167,7 @@ parse_frame_object( sizeof(void*), previous_frame ); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1212,7 +1215,7 @@ parse_async_frame_object( sizeof(void*), previous_frame ); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1242,6 +1245,7 @@ parse_async_frame_object( return -1; } + assert(code_object != NULL); if ((void*)*code_object == NULL) { return 0; } @@ -1261,7 +1265,8 @@ read_offsets( _Py_DebugOffsets* debug_offsets ) { *runtime_start_address = get_py_runtime(pid); - if (!*runtime_start_address) { + assert(runtime_start_address != NULL); + if ((void*)*runtime_start_address == NULL) { if (!PyErr_Occurred()) { PyErr_SetString( PyExc_RuntimeError, "Failed to get .PyRuntime address"); @@ -1271,7 +1276,7 @@ read_offsets( size_t size = sizeof(struct _Py_DebugOffsets); ssize_t bytes_read = read_memory( pid, *runtime_start_address, size, debug_offsets); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } return 0; @@ -1289,7 +1294,7 @@ read_async_debug( size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); ssize_t bytes_read = read_memory( pid, async_debug_addr, size, async_debug); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } return 0; @@ -1311,7 +1316,7 @@ find_running_frame( runtime_start_address + interpreter_state_list_head, sizeof(void*), &address_of_interpreter_state); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1327,7 +1332,7 @@ find_running_frame( local_debug_offsets->interpreter_state.threads_head, sizeof(void*), &address_of_thread); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1366,7 +1371,7 @@ find_running_task( runtime_start_address + interpreter_state_list_head, sizeof(void*), &address_of_interpreter_state); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } @@ -1382,7 +1387,7 @@ find_running_task( local_debug_offsets->interpreter_state.threads_head, sizeof(void*), &address_of_thread); - if (bytes_read == -1) { + if (bytes_read < 0) { return -1; } From d0aedf084d578df55050553752a868099539e585 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Fri, 22 Nov 2024 20:53:57 -0800 Subject: [PATCH 58/84] Address Kumar's latest comment --- Modules/_asynciomodule.c | 53 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 985b10c3e0eb8b9..aab57603a6a541a 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2136,9 +2136,49 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) return -1; } + assert(task == item); + Py_CLEAR(item); + + // This block is needed to enable `asyncio.capture_call_graph()` API. + // We want to be enable debuggers and profilers to be able to quickly + // introspect the asyncio running state from another process. + // When we do that, we need to essentially traverse the address space + // of a Python process and understand what every Python thread in it is + // currently doing, mainly: + // + // * current frame + // * current asyncio task + // + // A naive solution would be to require profilers and debuggers to + // find the current task in the "_asynciomodule" module state, but + // unfortunately that would require a lot of complicated remote + // memory reads and logic, as Python's dict is a notoriously complex + // and ever-changing data structure. + // + // So the actual solution is to put a reference to the currently + // running asyncio Task to the interpreter thread state (we already + // have some asyncio state there.) _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); - assert(ts->asyncio_running_task == NULL); - ts->asyncio_running_task = item; // strong ref + if (ts->asyncio_running_loop == loop) { + // Protect from a situation when someone calls this method + // from another thread. This shouldn't ever happen though, + // as `enter_task` and `leave_task` can either be called by: + // + // - `asyncio.Task` itself, in `Task.__step()`. That method + // can only be called by the event loop itself. + // + // - third-party Task "from scratch" implementations, that + // our `capture_call_graph` API doesn't support anyway. + // + // That said, we still want to make sure we don't end up in + // a broken state, so we check that we're in the correct thread + // by comparing the *loop* argument to the event loop set + // in the current thread. If they match we know we're in the + // right thread, as asyncio event loops don't change threads. + assert(ts->asyncio_running_task == NULL); + ts->asyncio_running_task = Py_NewRef(task); + } + return 0; } @@ -2171,9 +2211,14 @@ leave_task(asyncio_state *state, PyObject *loop, PyObject *task) return err_leave_task(Py_None, task); } + // See the comment in `enter_task` for the explanation of why + // the following is needed. _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); - Py_CLEAR(ts->asyncio_running_task); - return res; + if (ts->asyncio_running_loop == loop) { + Py_CLEAR(ts->asyncio_running_task); + } + + return 0; } static PyObject * From df0032a09e6a8e31dea9871901757b904a3188e7 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 12:45:21 +0530 Subject: [PATCH 59/84] diff cleanup --- Include/internal/pycore_runtime.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 7a4f94152c8f221..2f2cec22cf1589c 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -27,6 +27,7 @@ extern "C" { #include "pycore_typeobject.h" // struct _types_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_state + /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. From 0ce241bc29927529dc0d66a6458aa43ccabd55dc Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 12:59:52 +0530 Subject: [PATCH 60/84] change dataclasses to use tuples --- Lib/asyncio/graph.py | 27 +++++++++++++++------------ Lib/test/test_asyncio/test_graph.py | 4 ++-- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index fb613f034ba2d34..fc84531818ddfae 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -33,8 +33,8 @@ class FrameCallGraphEntry: @dataclasses.dataclass(frozen=True, slots=True) class FutureCallGraph: future: futures.Future - call_stack: list[FrameCallGraphEntry] - awaited_by: list[FutureCallGraph] + call_stack: tuple["FrameCallGraphEntry", ...] + awaited_by: tuple["FutureCallGraph", ...] def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: @@ -68,12 +68,13 @@ def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: awaited_by.append(_build_graph_for_future(parent)) st.reverse() - return FutureCallGraph(future, st, awaited_by) + return FutureCallGraph(future, tuple(st), tuple(awaited_by)) def capture_call_graph( - *, future: futures.Future | None = None, + /, + *, depth: int = 1, ) -> FutureCallGraph | None: """Capture async call graph for the current task or the provided Future. @@ -85,16 +86,16 @@ def capture_call_graph( Where 'future' is a reference to an asyncio.Future or asyncio.Task (or their subclasses.) - 'call_stack' is a list of FrameGraphEntry objects. + 'call_stack' is a tuple of FrameGraphEntry objects. - 'awaited_by' is a list of FutureCallGraph objects. + 'awaited_by' is a tuple of FutureCallGraph objects. * FrameCallGraphEntry(frame) Where 'frame' is a frame object of a regular Python function in the call stack. - Receives an optional keyword-only "future" argument. If not passed, + Receives an optional "future" argument. If not passed, the current task will be used. If there's no current task, the function returns None. @@ -154,12 +155,13 @@ def capture_call_graph( for parent in future._asyncio_awaited_by: awaited_by.append(_build_graph_for_future(parent)) - return FutureCallGraph(future, call_stack, awaited_by) + return FutureCallGraph(future, tuple(call_stack), tuple(awaited_by)) def format_call_graph( - *, future: futures.Future | None = None, + /, + *, depth: int = 1, ) -> str: """Return async call graph as a string for `future`. @@ -224,7 +226,7 @@ def add_line(line: str) -> None: for fut in st.awaited_by: render_level(fut, buf, level + 1) - graph = capture_call_graph(future=future, depth=depth + 1) + graph = capture_call_graph(future, depth=depth + 1) if graph is None: return @@ -238,10 +240,11 @@ def add_line(line: str) -> None: del graph def print_call_graph( - *, future: futures.Future | None = None, + /, + *, file: typing.TextIO | None = None, depth: int = 1, ) -> None: """Print async call graph for the current task or the provided Future.""" - print(format_call_graph(future=future, depth=depth), file=file) + print(format_call_graph(future, depth=depth), file=file) diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 9536451e0efa858..25046e4cc2c2924 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -39,9 +39,9 @@ def walk(s): return ret buf = io.StringIO() - asyncio.print_call_graph(future=fut, file=buf, depth=depth+1) + asyncio.print_call_graph(fut, file=buf, depth=depth+1) - stack = asyncio.capture_call_graph(future=fut, depth=depth) + stack = asyncio.capture_call_graph(fut, depth=depth) return walk(stack), buf.getvalue() From 8f126f67ec1d21a8f4efed32b51ccccd04cb4a7c Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:06:06 +0530 Subject: [PATCH 61/84] doc fixes --- Doc/library/asyncio-graph.rst | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 4ed5b2609b8b4d6..2da58d2b3353eb6 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -12,7 +12,7 @@ Stack Introspection ------------------------------------- asyncio has powerful runtime call graph introspection utilities -to trace the entire call graph of a running coroutine or task, or +to trace the entire call graph of a running *coroutine* or *task*, or a suspended *future*. These utilities and the underlying machinery can be used by users in their Python code or by external profilers and debuggers. @@ -20,12 +20,12 @@ and debuggers. .. versionadded:: next -.. function:: print_call_graph(*, future=None, file=None, depth=1) +.. function:: print_call_graph(future=None, /, *, file=None, depth=1) Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. - The function receives an optional keyword-only *future* argument. + The function receives an optional *future* argument. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. @@ -33,7 +33,7 @@ and debuggers. keyword-only *depth* argument can be used to skip the specified number of frames from top of the stack. - If *file* is not specified the function will print to :data:`sys.stdout`. + If *file* is omitted or ``None``, the function will print to :data:`sys.stdout`. **Example:** @@ -63,16 +63,16 @@ and debuggers. | File 'taskgroups.py', line 107, in async TaskGroup.__aexit__() | File 't2.py', line 7, in async main() -.. function:: format_call_graph(*, future=None, depth=1) +.. function:: format_call_graph(future=None, /, *, depth=1) Like :func:`print_call_graph`, but returns a string. -.. function:: capture_call_graph(*, future=None) +.. function:: capture_call_graph(future=None, /, *, depth=1) Capture the async call graph for the current task or the provided :class:`Task` or :class:`Future`. - The function receives an optional keyword-only *future* argument. + The function receives an optional *future* argument. If not passed, the current running task will be used. If there's no current task, the function returns ``None``. @@ -87,9 +87,9 @@ and debuggers. Where *future* is a reference to a :class:`Future` or a :class:`Task` (or their subclasses.) - ``call_stack`` is a list of ``FrameCallGraphEntry`` objects. + ``call_stack`` is a tuple of ``FrameCallGraphEntry`` objects. - ``awaited_by`` is a list of ``FutureCallGraph`` objects. + ``awaited_by`` is a tuple of ``FutureCallGraph`` objects. * ``FrameCallGraphEntry(frame)`` From 966d84e75a5ec7fae16af718fb8997ee8218de5e Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:08:13 +0530 Subject: [PATCH 62/84] remove reduntant headers include and add my name to whatsnew --- Doc/whatsnew/3.14.rst | 4 ++-- Include/internal/pycore_runtime_init.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 594ac367a6c9896..9af068c60c68dba 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -541,8 +541,8 @@ asyncio * :mod:`asyncio` has new utility functions for introspecting and printing the program's call graph: :func:`asyncio.capture_call_graph` and :func:`asyncio.print_call_graph`. - (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa - in :gh:`91048`.) + (Contributed by Yury Selivanov, Pablo Galindo Salgado, Łukasz Langa + and Kumar Aditya in :gh:`91048`.) io --- diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index e4d9fa2e987eaa7..8a8f47695fb8b0e 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -23,7 +23,6 @@ extern "C" { #include "pycore_runtime_init_generated.h" // _Py_bytes_characters_INIT #include "pycore_signal.h" // _signals_RUNTIME_INIT #include "pycore_tracemalloc.h" // _tracemalloc_runtime_state_INIT -#include "pycore_genobject.h" extern PyTypeObject _PyExc_MemoryError; From f56468af8b3789d0e20be30715c20ac1bb367641 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:19:14 +0530 Subject: [PATCH 63/84] improve comment --- Modules/_asynciomodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index aab57603a6a541a..362d55ad97a4bb6 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2155,8 +2155,8 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) // memory reads and logic, as Python's dict is a notoriously complex // and ever-changing data structure. // - // So the actual solution is to put a reference to the currently - // running asyncio Task to the interpreter thread state (we already + // So the easier solution is to put a strong reference of the currently + // running `asyncio.Task` to the interpreter thread state (we already // have some asyncio state there.) _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); if (ts->asyncio_running_loop == loop) { @@ -2172,7 +2172,7 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) // // That said, we still want to make sure we don't end up in // a broken state, so we check that we're in the correct thread - // by comparing the *loop* argument to the event loop set + // by comparing the *loop* argument to running event loop // in the current thread. If they match we know we're in the // right thread, as asyncio event loops don't change threads. assert(ts->asyncio_running_task == NULL); From 404b88aeb4c444dd77f3819c37a83d8396d1f07f Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:38:27 +0530 Subject: [PATCH 64/84] fix leave task --- Modules/_asynciomodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 362d55ad97a4bb6..e227fd6dec76f13 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2218,7 +2218,7 @@ leave_task(asyncio_state *state, PyObject *loop, PyObject *task) Py_CLEAR(ts->asyncio_running_task); } - return 0; + return res; } static PyObject * From 911fed869bb7bd353c934cb48ef76d00499c3741 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:58:15 +0530 Subject: [PATCH 65/84] fix external inspection on linux --- Modules/_testexternalinspection.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index d0fb3ca2736c6d8..b696d70716d592d 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -359,7 +359,7 @@ search_map_for_section(pid_t pid, const char* secname, const char* map) section_header_table[i].sh_name + 1 // "+1" accounts for the leading "." ); - this_sec_name += 1; + if (strcmp(secname, this_sec_name) == 0) { section = §ion_header_table[i]; break; From ab511a4f67ffd88f7709c1bc1990074e6e5b555d Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 13:59:10 +0530 Subject: [PATCH 66/84] minor format --- Modules/_asynciomodule.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index e227fd6dec76f13..287550f4b1662c3 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -103,19 +103,19 @@ typedef struct { #endif typedef struct _Py_AsyncioModuleDebugOffsets { - struct _asyncio_task_object { - uint64_t size; - uint64_t task_name; - uint64_t task_awaited_by; - uint64_t task_is_task; - uint64_t task_awaited_by_is_set; - uint64_t task_coro; - } asyncio_task_object; - struct _asyncio_thread_state { - uint64_t size; - uint64_t asyncio_running_loop; - uint64_t asyncio_running_task; - } asyncio_thread_state; + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; + struct _asyncio_thread_state { + uint64_t size; + uint64_t asyncio_running_loop; + uint64_t asyncio_running_task; + } asyncio_thread_state; } Py_AsyncioModuleDebugOffsets; GENERATE_DEBUG_SECTION(AsyncioDebug, Py_AsyncioModuleDebugOffsets AsyncioDebug) From c3c685ace22fe1c3839db254d76fcfc3de9c7e29 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 14:11:58 +0530 Subject: [PATCH 67/84] try to fix docs build --- Doc/whatsnew/3.14.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 9af068c60c68dba..2ddb7e6d9cc995f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -542,7 +542,7 @@ asyncio the program's call graph: :func:`asyncio.capture_call_graph` and :func:`asyncio.print_call_graph`. (Contributed by Yury Selivanov, Pablo Galindo Salgado, Łukasz Langa - and Kumar Aditya in :gh:`91048`.) + and Kumar Aditya in :gh:`91048`.) io --- From 785adebc8d2e06f11a44f1c33aab2636a33dd8d1 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Sat, 23 Nov 2024 16:15:16 +0530 Subject: [PATCH 68/84] Update Doc/whatsnew/3.14.rst --- Doc/whatsnew/3.14.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2ddb7e6d9cc995f..11cc90c0160a5f8 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -541,8 +541,8 @@ asyncio * :mod:`asyncio` has new utility functions for introspecting and printing the program's call graph: :func:`asyncio.capture_call_graph` and :func:`asyncio.print_call_graph`. - (Contributed by Yury Selivanov, Pablo Galindo Salgado, Łukasz Langa - and Kumar Aditya in :gh:`91048`.) + (Contributed by Yury Selivanov, Pablo Galindo Salgado and Łukasz Langa + in :gh:`91048`.) io --- From a577328873f3db1785d2f367659100a86550a01b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 23 Nov 2024 11:09:54 +0100 Subject: [PATCH 69/84] Match indentation with _asynciomodule.c after ab511a4f67ffd88f7709c1bc1990074e6e5b555d --- Modules/_testexternalinspection.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index b696d70716d592d..a7c3bf3cb7f65da 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -60,19 +60,19 @@ #endif struct _Py_AsyncioModuleDebugOffsets { - struct _asyncio_task_object { - uint64_t size; - uint64_t task_name; - uint64_t task_awaited_by; - uint64_t task_is_task; - uint64_t task_awaited_by_is_set; - uint64_t task_coro; - } asyncio_task_object; - struct _asyncio_thread_state { - uint64_t size; - uint64_t asyncio_running_loop; - uint64_t asyncio_running_task; - } asyncio_thread_state; + struct _asyncio_task_object { + uint64_t size; + uint64_t task_name; + uint64_t task_awaited_by; + uint64_t task_is_task; + uint64_t task_awaited_by_is_set; + uint64_t task_coro; + } asyncio_task_object; + struct _asyncio_thread_state { + uint64_t size; + uint64_t asyncio_running_loop; + uint64_t asyncio_running_task; + } asyncio_thread_state; }; #if defined(__APPLE__) && TARGET_OS_OSX From 064129a92ee045ac5b64ff817f7918052d92e112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 23 Nov 2024 11:10:40 +0100 Subject: [PATCH 70/84] Improve comments after f56468af8b3789d0e20be30715c20ac1bb367641 --- Modules/_asynciomodule.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 287550f4b1662c3..0e7a6917e28c9e0 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2155,8 +2155,8 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) // memory reads and logic, as Python's dict is a notoriously complex // and ever-changing data structure. // - // So the easier solution is to put a strong reference of the currently - // running `asyncio.Task` to the interpreter thread state (we already + // So the easier solution is to put a strong reference to the currently + // running `asyncio.Task` on the interpreter thread state (we already // have some asyncio state there.) _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); if (ts->asyncio_running_loop == loop) { @@ -2172,7 +2172,7 @@ enter_task(asyncio_state *state, PyObject *loop, PyObject *task) // // That said, we still want to make sure we don't end up in // a broken state, so we check that we're in the correct thread - // by comparing the *loop* argument to running event loop + // by comparing the *loop* argument to the event loop running // in the current thread. If they match we know we're in the // right thread, as asyncio event loops don't change threads. assert(ts->asyncio_running_task == NULL); From ce332d933d366a9519a78557bdc7fea9c68e3a2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 23 Nov 2024 11:47:17 +0100 Subject: [PATCH 71/84] Restore the Oxford comma --- Doc/whatsnew/3.14.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 11cc90c0160a5f8..594ac367a6c9896 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -541,7 +541,7 @@ asyncio * :mod:`asyncio` has new utility functions for introspecting and printing the program's call graph: :func:`asyncio.capture_call_graph` and :func:`asyncio.print_call_graph`. - (Contributed by Yury Selivanov, Pablo Galindo Salgado and Łukasz Langa + (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa in :gh:`91048`.) io From d6d943f09290311e983ce83303a6f014b71647c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 23 Nov 2024 13:13:56 +0100 Subject: [PATCH 72/84] Address remaining suggestions of Andrew Svetlov --- Doc/library/asyncio-graph.rst | 32 ++++++++++++++------ Lib/asyncio/graph.py | 55 ++++++++++++++++++++++++++--------- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 2da58d2b3353eb6..6fa7ac2029d88fb 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -3,9 +3,9 @@ .. _asyncio-graph: -=================== -Stack Introspection -=================== +======================== +Call Graph Introspection +======================== **Source code:** :source:`Lib/asyncio/graph.py` @@ -20,20 +20,31 @@ and debuggers. .. versionadded:: next -.. function:: print_call_graph(future=None, /, *, file=None, depth=1) +.. function:: print_call_graph(future=None, /, *, file=None, depth=1, limit=None) Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. + This function prints entries starting from the currently executing frame, + i.e. the top frame, and going down towards the invocation point. + The function receives an optional *future* argument. - If not passed, the current running task will be used. If there's no - current task, the function returns ``None``. + If not passed, the current running task will be used. If the function is called on *the current task*, the optional keyword-only *depth* argument can be used to skip the specified number of frames from top of the stack. - If *file* is omitted or ``None``, the function will print to :data:`sys.stdout`. + If the optional keyword-only *limit* argument is provided, each call stack + in the resulting graph is truncated to include at most ``abs(limit)`` + entries. If *limit* is positive, the entries left are the closest to + the invocation point. If *limit* is negative, the topmost entries are + left. If *limit* is omitted or ``None``, all entries are present. + If *limit* is ``0``, the call stack is not printed at all, only + "awaited by" information is printed. + + If *file* is omitted or ``None``, the function will print + to :data:`sys.stdout`. **Example:** @@ -63,11 +74,14 @@ and debuggers. | File 'taskgroups.py', line 107, in async TaskGroup.__aexit__() | File 't2.py', line 7, in async main() -.. function:: format_call_graph(future=None, /, *, depth=1) +.. function:: format_call_graph(future=None, /, *, depth=1, limit=None) Like :func:`print_call_graph`, but returns a string. + If *future* is ``None`` and there's no current task, + the function returns an empty string. + -.. function:: capture_call_graph(future=None, /, *, depth=1) +.. function:: capture_call_graph(future=None, /, *, depth=1, limit=None) Capture the async call graph for the current task or the provided :class:`Task` or :class:`Future`. diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index fc84531818ddfae..95592ddadeeff02 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -37,7 +37,11 @@ class FutureCallGraph: awaited_by: tuple["FutureCallGraph", ...] -def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: +def _build_graph_for_future( + future: futures.Future, + *, + limit: int | None = None, +) -> FutureCallGraph: if not isinstance(future, futures.Future): raise TypeError( f"{future!r} object does not appear to be compatible " @@ -46,7 +50,7 @@ def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: coro = None if get_coro := getattr(future, 'get_coro', None): - coro = get_coro() + coro = get_coro() if limit != 0 else None st: list[FrameCallGraphEntry] = [] awaited_by: list[FutureCallGraph] = [] @@ -65,8 +69,13 @@ def _build_graph_for_future(future: futures.Future) -> FutureCallGraph: if future._asyncio_awaited_by: for parent in future._asyncio_awaited_by: - awaited_by.append(_build_graph_for_future(parent)) + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + if limit is not None: + if limit > 0: + st = st[:limit] + elif limit < 0: + st = st[limit:] st.reverse() return FutureCallGraph(future, tuple(st), tuple(awaited_by)) @@ -76,8 +85,9 @@ def capture_call_graph( /, *, depth: int = 1, + limit: int | None = None, ) -> FutureCallGraph | None: - """Capture async call graph for the current task or the provided Future. + """Capture the async call graph for the current task or the provided Future. The graph is represented with three data structures: @@ -95,13 +105,21 @@ def capture_call_graph( Where 'frame' is a frame object of a regular Python function in the call stack. - Receives an optional "future" argument. If not passed, + Receives an optional 'future' argument. If not passed, the current task will be used. If there's no current task, the function returns None. If "capture_call_graph()" is introspecting *the current task*, the - optional keyword-only "depth" argument can be used to skip the specified + optional keyword-only 'depth' argument can be used to skip the specified number of frames from top of the stack. + + If the optional keyword-only 'limit' argument is provided, each call stack + in the resulting graph is truncated to include at most ``abs(limit)`` + entries. If 'limit' is positive, the entries left are the closest to + the invocation point. If 'limit' is negative, the topmost entries are + left. If 'limit' is omitted or None, all entries are present. + If 'limit' is 0, the call stack is not captured at all, only + "awaited by" information is present. """ loop = events._get_running_loop() @@ -111,7 +129,7 @@ def capture_call_graph( # if yes - check if the passed future is the currently # running task or not. if loop is None or future is not tasks.current_task(loop=loop): - return _build_graph_for_future(future) + return _build_graph_for_future(future, limit=limit) # else: future is the current task, move on. else: if loop is None: @@ -134,7 +152,7 @@ def capture_call_graph( call_stack: list[FrameCallGraphEntry] = [] - f = sys._getframe(depth) + f = sys._getframe(depth) if limit != 0 else None try: while f is not None: is_async = f.f_generator is not None @@ -153,7 +171,14 @@ def capture_call_graph( awaited_by = [] if future._asyncio_awaited_by: for parent in future._asyncio_awaited_by: - awaited_by.append(_build_graph_for_future(parent)) + awaited_by.append(_build_graph_for_future(parent, limit=limit)) + + if limit is not None: + limit *= -1 + if limit > 0: + call_stack = call_stack[:limit] + elif limit < 0: + call_stack = call_stack[limit:] return FutureCallGraph(future, tuple(call_stack), tuple(awaited_by)) @@ -163,8 +188,9 @@ def format_call_graph( /, *, depth: int = 1, + limit: int | None = None, ) -> str: - """Return async call graph as a string for `future`. + """Return the async call graph as a string for `future`. If `future` is not provided, format the call graph for the current task. """ @@ -226,9 +252,9 @@ def add_line(line: str) -> None: for fut in st.awaited_by: render_level(fut, buf, level + 1) - graph = capture_call_graph(future, depth=depth + 1) + graph = capture_call_graph(future, depth=depth + 1, limit=limit) if graph is None: - return + return "" try: buf: list[str] = [] @@ -245,6 +271,7 @@ def print_call_graph( *, file: typing.TextIO | None = None, depth: int = 1, + limit: int | None = None, ) -> None: - """Print async call graph for the current task or the provided Future.""" - print(format_call_graph(future, depth=depth), file=file) + """Print the async call graph for the current task or the provided Future.""" + print(format_call_graph(future, depth=depth, limit=limit), file=file) From 703ff4668e4b01b4ece27300c3e770301572db33 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Sat, 23 Nov 2024 14:26:57 -0800 Subject: [PATCH 73/84] Fix gather() --- Lib/asyncio/tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 4025163416cde36..d1587c59cc57280 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -887,13 +887,13 @@ def _done_callback(fut, cur_task): # can't control it, disable the "destroy pending task" # warning. fut._log_destroy_pending = False - if cur_task is not None: - futures.future_add_to_awaited_by(fut, cur_task) nfuts += 1 arg_to_fut[arg] = fut if fut.done(): done_futs.append(fut) else: + if cur_task is not None: + futures.future_add_to_awaited_by(fut, cur_task) fut.add_done_callback(lambda fut: _done_callback(fut, cur_task)) else: From e8678632077a75f9ad27edae3bc4ea4fcd72d8dd Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 26 Nov 2024 00:17:52 +0000 Subject: [PATCH 74/84] Replace lambda by closure --- Lib/asyncio/tasks.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index d1587c59cc57280..a25854cc4bd69eb 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -807,7 +807,13 @@ def gather(*coros_or_futures, return_exceptions=False): outer.set_result([]) return outer - def _done_callback(fut, cur_task): + loop = events._get_running_loop() + if loop is not None: + cur_task = current_task(loop) + else: + cur_task = None + + def _done_callback(fut, cur_task=cur_task): nonlocal nfinished nfinished += 1 @@ -871,11 +877,6 @@ def _done_callback(fut, cur_task): nfinished = 0 done_futs = [] outer = None # bpo-46672 - loop = events._get_running_loop() - if loop is not None: - cur_task = current_task(loop) - else: - cur_task = None for arg in coros_or_futures: if arg not in arg_to_fut: fut = ensure_future(arg, loop=loop) @@ -894,7 +895,7 @@ def _done_callback(fut, cur_task): else: if cur_task is not None: futures.future_add_to_awaited_by(fut, cur_task) - fut.add_done_callback(lambda fut: _done_callback(fut, cur_task)) + fut.add_done_callback(_done_callback) else: # There's a duplicate Future object in coros_or_futures. @@ -909,7 +910,7 @@ def _done_callback(fut, cur_task): # this will effectively complete the gather eagerly, with the last # callback setting the result (or exception) on outer before returning it for fut in done_futs: - _done_callback(fut, cur_task) + _done_callback(fut) return outer From 9cb5b2900e46baff89afb696ce5482bc702cd0bc Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Mon, 25 Nov 2024 18:24:56 -0600 Subject: [PATCH 75/84] =?UTF-8?q?Let=E2=80=99s=20not=20emphasize=20*asynci?= =?UTF-8?q?o*?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jacob Coffee --- Doc/library/asyncio-graph.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 6fa7ac2029d88fb..010d2061c8e2d56 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -118,7 +118,7 @@ To introspect an async call graph asyncio requires cooperation from control flow structures, such as :func:`shield` or :class:`TaskGroup`. Any time an intermediate :class:`Future` object with low-level APIs like :meth:`Future.add_done_callback() ` is -involved, the following two functions should be used to inform *asyncio* +involved, the following two functions should be used to inform asyncio about how exactly such intermediate future objects are connected with the tasks they wrap or control. From 61b2b7b87050ec5814716d7caa663867e44d9640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 26 Nov 2024 11:51:19 +0100 Subject: [PATCH 76/84] Apply suggestions from Irit's review Co-authored-by: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> --- Doc/library/asyncio-graph.rst | 2 +- Lib/asyncio/graph.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 010d2061c8e2d56..221438f9b85cf8c 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -14,7 +14,7 @@ Call Graph Introspection asyncio has powerful runtime call graph introspection utilities to trace the entire call graph of a running *coroutine* or *task*, or a suspended *future*. These utilities and the underlying machinery -can be used by users in their Python code or by external profilers +can be used from within a Python program or by external profilers and debuggers. .. versionadded:: next diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index 95592ddadeeff02..5914a329945f701 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -17,7 +17,7 @@ 'FutureCallGraph', ) -# Sadly, we can't re-use the traceback's module datastructures as those +# Sadly, we can't re-use the traceback module's datastructures as those # are tailored for error reporting, whereas we need to represent an # async call graph. # @@ -93,8 +93,7 @@ def capture_call_graph( * FutureCallGraph(future, call_stack, awaited_by) - Where 'future' is a reference to an asyncio.Future or asyncio.Task - (or their subclasses.) + Where 'future' is an instance of asyncio.Future or asyncio.Task. 'call_stack' is a tuple of FrameGraphEntry objects. @@ -256,14 +255,14 @@ def add_line(line: str) -> None: if graph is None: return "" + buf: list[str] = [] try: - buf: list[str] = [] render_level(graph, buf, 0) - return '\n'.join(buf) finally: # 'graph' has references to frames so we should # make sure it's GC'ed as soon as we don't need it. del graph + return '\n'.join(buf) def print_call_graph( future: futures.Future | None = None, From 9533ab922ec3c2e81b9bba95d6b4f154512546a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 26 Nov 2024 14:00:06 +0100 Subject: [PATCH 77/84] Address remaining review by Irit --- Doc/library/asyncio-graph.rst | 3 +-- Doc/library/inspect.rst | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 221438f9b85cf8c..b39f5d8a22455c4 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -25,8 +25,7 @@ and debuggers. Print the async call graph for the current task or the provided :class:`Task` or :class:`Future`. - This function prints entries starting from the currently executing frame, - i.e. the top frame, and going down towards the invocation point. + This function prints entries starting from the top frame and going The function receives an optional *future* argument. If not passed, the current running task will be used. diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 0902d64f9bd22a5..4c755a9ecabf4b1 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -150,6 +150,12 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | f_locals | local namespace seen by | | | | this frame | +-----------------+-------------------+---------------------------+ +| | f_generator | returns the generator or | +| | | coroutine object that | +| | | owns this frame, or | +| | | ``None`` if the frame is | +| | | of a regular function | ++-----------------+-------------------+---------------------------+ | | f_trace | tracing function for this | | | | frame, or ``None`` | +-----------------+-------------------+---------------------------+ @@ -162,12 +168,6 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | per-opcode events are | | | | requested | +-----------------+-------------------+---------------------------+ -| | f_generator | returns the generator or | -| | | coroutine object that | -| | | owns this frame, or | -| | | ``None`` if the frame is | -| | | of a regular function | -+-----------------+-------------------+---------------------------+ | | clear() | used to clear all | | | | references to local | | | | variables | From 596191d17220a5c047879914294ecdf85cea1f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 26 Nov 2024 14:38:38 +0100 Subject: [PATCH 78/84] Test pure-Python and C-accelerated versions of future_add_to/future_discard_from --- Lib/asyncio/futures.py | 13 ++-- Lib/test/test_asyncio/test_graph.py | 100 ++++++++++++++++++++++++++-- 2 files changed, 103 insertions(+), 10 deletions(-) diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index 71fd283acfa5633..a1c97c5ae706b3e 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -465,6 +465,9 @@ def future_discard_from_awaited_by(fut, waiter, /): fut._Future__asyncio_awaited_by.discard(waiter) +_py_future_add_to_awaited_by = future_add_to_awaited_by +_py_future_discard_from_awaited_by = future_discard_from_awaited_by + try: import _asyncio except ImportError: @@ -472,9 +475,7 @@ def future_discard_from_awaited_by(fut, waiter, /): else: # _CFuture is needed for tests. Future = _CFuture = _asyncio.Future - -try: - from _asyncio import future_add_to_awaited_by, \ - future_discard_from_awaited_by -except ImportError: - pass + future_add_to_awaited_by = _asyncio.future_add_to_awaited_by + future_discard_from_awaited_by = _asyncio.future_discard_from_awaited_by + _c_future_add_to_awaited_by = future_add_to_awaited_by + _c_future_discard_from_awaited_by = future_discard_from_awaited_by diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 25046e4cc2c2924..9e984e2499bc1b7 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -45,8 +45,7 @@ def walk(s): return walk(stack), buf.getvalue() -class TestCallStack(unittest.IsolatedAsyncioTestCase): - +class CallStackTestBase: async def test_stack_tgroup(self): @@ -107,7 +106,7 @@ async def main(): ]) self.assertIn( - ' async TestCallStack.test_stack_tgroup()', + ' async CallStackTestBase.test_stack_tgroup()', stack_for_c5[1]) @@ -143,8 +142,11 @@ async def main(): [] ]) + from pprint import pprint + pprint(stack_for_gen_nested_call[1]) + self.assertIn( - 'async generator TestCallStack.test_stack_async_gen..gen()', + 'async generator CallStackTestBase.test_stack_async_gen..gen()', stack_for_gen_nested_call[1]) async def test_stack_gather(self): @@ -345,3 +347,93 @@ async def main(): ) self.assertTrue(stack_for_fut[1].startswith('* Future(id=')) + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_c_future_add_to_awaited_by"), + "C-accelerated asyncio call graph backend missing", +) +class TestCallStackC(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._CFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._CTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._c_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._c_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future + + +@unittest.skipIf( + not hasattr(asyncio.futures, "_py_future_add_to_awaited_by"), + "Pure Python asyncio call graph backend missing", +) +class TestCallStackPy(CallStackTestBase, unittest.IsolatedAsyncioTestCase): + def setUp(self): + futures = asyncio.futures + tasks = asyncio.tasks + + self._Future = asyncio.Future + asyncio.Future = futures.Future = futures._PyFuture + + self._Task = asyncio.Task + asyncio.Task = tasks.Task = tasks._PyTask + + self._future_add_to_awaited_by = asyncio.future_add_to_awaited_by + futures.future_add_to_awaited_by = futures._py_future_add_to_awaited_by + asyncio.future_add_to_awaited_by = futures.future_add_to_awaited_by + + self._future_discard_from_awaited_by = asyncio.future_discard_from_awaited_by + futures.future_discard_from_awaited_by = futures._py_future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = futures.future_discard_from_awaited_by + + + def tearDown(self): + futures = asyncio.futures + tasks = asyncio.tasks + + futures.future_discard_from_awaited_by = self._future_discard_from_awaited_by + asyncio.future_discard_from_awaited_by = self._future_discard_from_awaited_by + del self._future_discard_from_awaited_by + + futures.future_add_to_awaited_by = self._future_add_to_awaited_by + asyncio.future_add_to_awaited_by = self._future_add_to_awaited_by + del self._future_add_to_awaited_by + + asyncio.Task = self._Task + tasks = self._Task + del self._Task + + asyncio.Future = self._Future + futures.Future = self._Future + del self._Future From ad9152eb0d0fcb91f96524bdb4647bc3b3d26c5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 26 Nov 2024 14:44:17 +0100 Subject: [PATCH 79/84] fix blooper --- Lib/test/test_asyncio/test_graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 9e984e2499bc1b7..97e1a48564d14c2 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -386,7 +386,7 @@ def tearDown(self): del self._future_add_to_awaited_by asyncio.Task = self._Task - tasks = self._Task + tasks.Task = self._Task del self._Task asyncio.Future = self._Future @@ -431,7 +431,7 @@ def tearDown(self): del self._future_add_to_awaited_by asyncio.Task = self._Task - tasks = self._Task + tasks.Task = self._Task del self._Task asyncio.Future = self._Future From 066bf210ee388b0a7a046d0ee64015da05713485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 26 Nov 2024 14:45:45 +0100 Subject: [PATCH 80/84] Fix another blooper --- Doc/library/asyncio-graph.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index b39f5d8a22455c4..fc8edeb426c567b 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -26,6 +26,7 @@ and debuggers. :class:`Task` or :class:`Future`. This function prints entries starting from the top frame and going + down towards the invocation point. The function receives an optional *future* argument. If not passed, the current running task will be used. From 4caeec408cc085c378f01b5913bd4e526868c115 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 21 Jan 2025 20:39:03 +0000 Subject: [PATCH 81/84] Fix crash --- Modules/_asynciomodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 0e7a6917e28c9e0..3ebc6b65052324d 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2214,7 +2214,7 @@ leave_task(asyncio_state *state, PyObject *loop, PyObject *task) // See the comment in `enter_task` for the explanation of why // the following is needed. _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); - if (ts->asyncio_running_loop == loop) { + if (ts->asyncio_running_loop == NULL || ts->asyncio_running_loop == loop) { Py_CLEAR(ts->asyncio_running_task); } From a8dd667b26732f0d9af2993a604aaa88fed94603 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 22 Jan 2025 00:43:12 +0000 Subject: [PATCH 82/84] use private method for the policy --- Lib/test/test_asyncio/test_graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 97e1a48564d14c2..4b9d0bf5b470c5e 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -5,7 +5,7 @@ # To prevent a warning "test altered the execution environment" def tearDownModule(): - asyncio.set_event_loop_policy(None) + asyncio._set_event_loop_policy(None) def capture_test_stack(*, fut=None, depth=1): From cf8f5e569bdd0bdb4d6e6fc1875c040c7cfffeea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 22 Jan 2025 14:53:31 +0100 Subject: [PATCH 83/84] Avoid importing typing --- Lib/asyncio/graph.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index 5914a329945f701..d8df7c9919abbf7 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -3,7 +3,6 @@ import dataclasses import sys import types -import typing from . import events from . import futures @@ -17,6 +16,9 @@ 'FutureCallGraph', ) +if False: # for type checkers + from typing import TextIO + # Sadly, we can't re-use the traceback module's datastructures as those # are tailored for error reporting, whereas we need to represent an # async call graph. @@ -268,7 +270,7 @@ def print_call_graph( future: futures.Future | None = None, /, *, - file: typing.TextIO | None = None, + file: TextIO | None = None, depth: int = 1, limit: int | None = None, ) -> None: From eda9c7cb2ef3003e24fa65913750ede09ef7590c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 22 Jan 2025 15:00:46 +0100 Subject: [PATCH 84/84] Remove debug printing from test_asyncio.test_graph --- Lib/test/test_asyncio/test_graph.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 4b9d0bf5b470c5e..fd2160d4ca31374 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -142,9 +142,6 @@ async def main(): [] ]) - from pprint import pprint - pprint(stack_for_gen_nested_call[1]) - self.assertIn( 'async generator CallStackTestBase.test_stack_async_gen..gen()', stack_for_gen_nested_call[1])