Merge pull request #399 from goodboy/oob_cancel_testing

OoB (out-of-band) cancellation testing, proper.

examples/advanced_faults/ipc_failure_during_stream.py

@@ -16,6 +16,7 @@ from tractor import (
     ContextCancelled,
     MsgStream,
     _testing,
+    trionics,
 )
 import trio
 import pytest
@@ -62,9 +63,8 @@ async def recv_and_spawn_net_killers(
     await ctx.started()
     async with (
         ctx.open_stream() as stream,
-        trio.open_nursery(
-            strict_exception_groups=False,
-        ) as tn,
+        trionics.collapse_eg(),
+        trio.open_nursery() as tn,
     ):
         async for i in stream:
             print(f'child echoing {i}')

examples/debugging/asyncio_bp.py

@@ -29,7 +29,7 @@ async def bp_then_error(
     to_trio.send_nowait('start')
 
     # NOTE: what happens here inside the hook needs some refinement..
-    # => seems like it's still `._debug._set_trace()` but
+    # => seems like it's still `.debug._set_trace()` but
     #    we set `Lock.local_task_in_debug = 'sync'`, we probably want
     #    some further, at least, meta-data about the task/actor in debug
     #    in terms of making it clear it's `asyncio` mucking about.

examples/debugging/restore_builtin_breakpoint.py

@@ -5,7 +5,7 @@ import trio
 import tractor
 
 # ensure mod-path is correct!
-from tractor.devx._debug import (
+from tractor.devx.debug import (
     _sync_pause_from_builtin as _sync_pause_from_builtin,
 )
 
@@ -19,19 +19,22 @@ async def main() -> None:
     async with tractor.open_nursery(
         debug_mode=True,
         loglevel='devx',
-    ) as an:
-        assert an
+        maybe_enable_greenback=True,
+        # ^XXX REQUIRED to enable `breakpoint()` support (from sync
+        # fns) and thus required here to avoid an assertion err
+        # on the next line
+    ):
         assert (
             (pybp_var := os.environ['PYTHONBREAKPOINT'])
             ==
-            'tractor.devx._debug._sync_pause_from_builtin'
+            'tractor.devx.debug._sync_pause_from_builtin'
         )
 
         # TODO: an assert that verifies the hook has indeed been, hooked
         # XD
         assert (
             (pybp_hook := sys.breakpointhook)
-            is not tractor.devx._debug._set_trace
+            is not tractor.devx.debug._set_trace
         )
 
         print(

examples/debugging/root_self_cancelled_w_error.py

@@ -0,0 +1,35 @@
+import trio
+import tractor
+
+
+async def main():
+    async with tractor.open_root_actor(
+        debug_mode=True,
+        loglevel='cancel',
+    ) as _root:
+
+        # manually trigger self-cancellation and wait
+        # for it to fully trigger.
+        _root.cancel_soon()
+        await _root._cancel_complete.wait()
+        print('root cancelled')
+
+        # now ensure we can still use the REPL
+        try:
+            await tractor.pause()
+        except trio.Cancelled as _taskc:
+            assert (root_cs := _root._root_tn.cancel_scope).cancel_called
+            # NOTE^^ above logic but inside `open_root_actor()` and
+            # passed to the `shield=` expression is effectively what
+            # we're testing here!
+            await tractor.pause(shield=root_cs.cancel_called)
+
+        # XXX, if shield logic *is wrong* inside `open_root_actor()`'s
+        # crash-handler block this should never be interacted,
+        # instead `trio.Cancelled` would be bubbled up: the original
+        # BUG.
+        assert 0
+
+
+if __name__ == '__main__':
+    trio.run(main)

examples/debugging/subactor_bp_in_ctx.py

@@ -33,8 +33,11 @@ async def just_bp(
 
 
 async def main():
+
     async with tractor.open_nursery(
         debug_mode=True,
+        enable_transports=['uds'],
+        loglevel='devx',
     ) as n:
         p = await n.start_actor(
             'bp_boi',

examples/debugging/sync_bp.py

@@ -6,7 +6,7 @@ import tractor
 
 # TODO: only import these when not running from test harness?
 # can we detect `pexpect` usage maybe?
-# from tractor.devx._debug import (
+# from tractor.devx.debug import (
 #     get_lock,
 #     get_debug_req,
 # )

examples/quick_cluster.py

@@ -23,9 +23,8 @@ async def main():
             modules=[__name__]
         ) as portal_map,
 
-        trio.open_nursery(
-            strict_exception_groups=False,
-        ) as tn,
+        tractor.trionics.collapse_eg(),
+        trio.open_nursery() as tn,
     ):
 
         for (name, portal) in portal_map.items():

examples/trio/lockacquire_not_unmasked.py

@@ -0,0 +1,85 @@
+from contextlib import (
+    asynccontextmanager as acm,
+)
+from functools import partial
+
+import tractor
+import trio
+
+
+log = tractor.log.get_logger(
+    name=__name__
+)
+
+_lock: trio.Lock|None = None
+
+
+@acm
+async def acquire_singleton_lock(
+) -> None:
+    global _lock
+    if _lock is None:
+        log.info('Allocating LOCK')
+        _lock = trio.Lock()
+
+    log.info('TRYING TO LOCK ACQUIRE')
+    async with _lock:
+        log.info('ACQUIRED')
+        yield _lock
+
+    log.info('RELEASED')
+
+
+
+async def hold_lock_forever(
+    task_status=trio.TASK_STATUS_IGNORED
+):
+    async with (
+        tractor.trionics.maybe_raise_from_masking_exc(),
+        acquire_singleton_lock() as lock,
+    ):
+        task_status.started(lock)
+        await trio.sleep_forever()
+
+
+async def main(
+    ignore_special_cases: bool,
+    loglevel: str = 'info',
+    debug_mode: bool = True,
+):
+    async with (
+        trio.open_nursery() as tn,
+
+        # tractor.trionics.maybe_raise_from_masking_exc()
+        # ^^^ XXX NOTE, interestingly putting the unmasker
+        # here does not exhibit the same behaviour ??
+    ):
+        if not ignore_special_cases:
+            from tractor.trionics import _taskc
+            _taskc._mask_cases.clear()
+
+        _lock = await tn.start(
+            hold_lock_forever,
+        )
+        with trio.move_on_after(0.2):
+            await tn.start(
+                hold_lock_forever,
+            )
+
+        tn.cancel_scope.cancel()
+
+
+# XXX, manual test as script
+if __name__ == '__main__':
+    tractor.log.get_console_log(level='info')
+    for case in [True, False]:
+        log.info(
+            f'\n'
+            f'------ RUNNING SCRIPT TRIAL ------\n'
+            f'ignore_special_cases: {case!r}\n'
+        )
+        trio.run(partial(
+            main,
+            ignore_special_cases=case,
+            loglevel='info',
+        ))

examples/trio/send_chan_aclose_masks_beg.py

@@ -0,0 +1,195 @@
+from contextlib import (
+    contextmanager as cm,
+    # TODO, any diff in async case(s)??
+    # asynccontextmanager as acm,
+)
+from functools import partial
+
+import tractor
+import trio
+
+
+log = tractor.log.get_logger(
+    name=__name__
+)
+
+
+@cm
+def teardown_on_exc(
+    raise_from_handler: bool = False,
+):
+    '''
+    You could also have a teardown handler which catches any exc and
+    does some required teardown. In this case the problem is
+    compounded UNLESS you ensure the handler's scope is OUTSIDE the
+    `ux.aclose()`.. that is in the caller's enclosing scope.
+
+    '''
+    try:
+        yield
+    except BaseException as _berr:
+        berr = _berr
+        log.exception(
+            f'Handling termination teardown in child due to,\n'
+            f'{berr!r}\n'
+        )
+        if raise_from_handler:
+            # XXX teardown ops XXX
+            # on termination these steps say need to be run to
+            # ensure wider system consistency (like the state of
+            # remote connections/services).
+            #
+            # HOWEVER, any bug in this teardown code is also
+            # masked by the `tx.aclose()`!
+            # this is also true if `_tn.cancel_scope` is
+            # `.cancel_called` by the parent in a graceful
+            # request case..
+
+            # simulate a bug in teardown handler.
+            raise RuntimeError(
+                'woopsie teardown bug!'
+            )
+
+        raise  # no teardown bug.
+
+
+async def finite_stream_to_rent(
+    tx: trio.abc.SendChannel,
+    child_errors_mid_stream: bool,
+    raise_unmasked: bool,
+
+    task_status: trio.TaskStatus[
+        trio.CancelScope,
+    ] = trio.TASK_STATUS_IGNORED,
+):
+    async with (
+        # XXX without this unmasker the mid-streaming RTE is never
+        # reported since it is masked by the `tx.aclose()`
+        # call which in turn raises `Cancelled`!
+        #
+        # NOTE, this is WITHOUT doing any exception handling
+        # inside the child  task!
+        #
+        # TODO, uncomment next LoC to see the supprsessed beg[RTE]!
+        tractor.trionics.maybe_raise_from_masking_exc(
+            raise_unmasked=raise_unmasked,
+        ),
+
+        tx as tx,  # .aclose() is the guilty masker chkpt!
+
+        # XXX, this ONLY matters in the
+        # `child_errors_mid_stream=False` case oddly!?
+        # THAT IS, if no tn is opened in that case then the
+        # test will not fail; it raises the RTE correctly?
+        #
+        # -> so it seems this new scope somehow affects the form of
+        #    eventual in the parent EG?
+        tractor.trionics.maybe_open_nursery(
+            nursery=(
+                None
+                if not child_errors_mid_stream
+                else True
+            ),
+        ) as _tn,
+    ):
+        # pass our scope back to parent for supervision\
+        # control.
+        cs: trio.CancelScope|None = (
+            None
+            if _tn is True
+            else _tn.cancel_scope
+        )
+        task_status.started(cs)
+
+        with teardown_on_exc(
+            raise_from_handler=not child_errors_mid_stream,
+        ):
+            for i in range(100):
+                log.debug(
+                    f'Child tx {i!r}\n'
+                )
+                if (
+                    child_errors_mid_stream
+                    and
+                    i == 66
+                ):
+                    # oh wait but WOOPS there's a bug
+                    # in that teardown code!?
+                    raise RuntimeError(
+                        'woopsie, a mid-streaming bug!?'
+                    )
+
+                await tx.send(i)
+
+
+async def main(
+    # TODO! toggle this for the 2 cases!
+    # 1. child errors mid-stream while parent is also requesting
+    #   (graceful) cancel of that child streamer.
+    #
+    # 2. child contains a teardown handler which contains a
+    #   bug and raises.
+    #
+    child_errors_mid_stream: bool,
+
+    raise_unmasked: bool = False,
+    loglevel: str = 'info',
+):
+    tractor.log.get_console_log(level=loglevel)
+
+    # the `.aclose()` being checkpoints on these
+    # is the source of the problem..
+    tx, rx = trio.open_memory_channel(1)
+
+    async with (
+        tractor.trionics.collapse_eg(),
+        trio.open_nursery() as tn,
+        rx as rx,
+    ):
+        _child_cs = await tn.start(
+            partial(
+                finite_stream_to_rent,
+                child_errors_mid_stream=child_errors_mid_stream,
+                raise_unmasked=raise_unmasked,
+                tx=tx,
+            )
+        )
+        async for msg in rx:
+            log.debug(
+                f'Rent rx {msg!r}\n'
+            )
+
+            # simulate some external cancellation
+            # request **JUST BEFORE** the child errors.
+            if msg == 65:
+                log.cancel(
+                    f'Cancelling parent on,\n'
+                    f'msg={msg}\n'
+                    f'\n'
+                    f'Simulates OOB cancel request!\n'
+                )
+                tn.cancel_scope.cancel()
+
+
+# XXX, manual test as script
+if __name__ == '__main__':
+    tractor.log.get_console_log(level='info')
+    for case in [True, False]:
+        log.info(
+            f'\n'
+            f'------ RUNNING SCRIPT TRIAL ------\n'
+            f'child_errors_midstream: {case!r}\n'
+        )
+        try:
+            trio.run(partial(
+                main,
+                child_errors_mid_stream=case,
+                # raise_unmasked=True,
+                loglevel='info',
+            ))
+        except Exception as _exc:
+            exc = _exc
+            log.exception(
+                'Should have raised an RTE or Cancelled?\n'
+            )
+            breakpoint()

pyproject.toml

@@ -63,6 +63,7 @@ dev = [
   "stackscope>=0.2.2,<0.3",
   # ^ requires this?
   "typing-extensions>=4.14.1",
+
   "pyperclip>=1.9.0",
   "prompt-toolkit>=3.0.50",
   "xonsh>=0.19.2",

tests/conftest.py

@@ -103,7 +103,7 @@ def sig_prog(
 def daemon(
     debug_mode: bool,
     loglevel: str,
-    testdir,
+    testdir: pytest.Pytester,
     reg_addr: tuple[str, int],
     tpt_proto: str,
 

tests/devx/conftest.py

@@ -2,9 +2,11 @@
 `tractor.devx.*` tooling sub-pkg test space.
 
 '''
+from __future__ import annotations
 import time
 from typing import (
     Callable,
+    TYPE_CHECKING,
 )
 
 import pytest
@@ -16,7 +18,7 @@ from pexpect.spawnbase import SpawnBase
 from tractor._testing import (
     mk_cmd,
 )
-from tractor.devx._debug import (
+from tractor.devx.debug import (
     _pause_msg as _pause_msg,
     _crash_msg as _crash_msg,
     _repl_fail_msg as _repl_fail_msg,
@@ -26,14 +28,22 @@ from ..conftest import (
     _ci_env,
 )
 
+if TYPE_CHECKING:
+    from pexpect import pty_spawn
+
+
+# a fn that sub-instantiates a `pexpect.spawn()`
+# and returns it.
+type PexpectSpawner = Callable[[str], pty_spawn.spawn]
+
 
 @pytest.fixture
 def spawn(
-    start_method,
+    start_method: str,
     testdir: pytest.Pytester,
     reg_addr: tuple[str, int],
 
-) -> Callable[[str], None]:
+) -> PexpectSpawner:
     '''
     Use the `pexpect` module shipped via `testdir.spawn()` to
     run an `./examples/..` script by name.
@@ -59,7 +69,7 @@ def spawn(
     def _spawn(
         cmd: str,
         **mkcmd_kwargs,
-    ):
+    ) -> pty_spawn.spawn:
         unset_colors()
         return testdir.spawn(
             cmd=mk_cmd(
@@ -73,7 +83,7 @@ def spawn(
         )
 
     # such that test-dep can pass input script name.
-    return _spawn
+    return _spawn  # the `PexpectSpawner`, type alias.
 
 
 @pytest.fixture(
@@ -111,7 +121,7 @@ def ctlc(
         # XXX: disable pygments highlighting for auto-tests
         # since some envs (like actions CI) will struggle
         # the the added color-char encoding..
-        from tractor.devx._debug import TractorConfig
+        from tractor.devx.debug import TractorConfig
         TractorConfig.use_pygements = False
 
     yield use_ctlc

tests/devx/test_debugger.py

@@ -1,19 +1,23 @@
 """
 That "native" debug mode better work!
 
-All these tests can be understood (somewhat) by running the equivalent
-`examples/debugging/` scripts manually.
+All these tests can be understood (somewhat) by running the
+equivalent `examples/debugging/` scripts manually.
 
 TODO:
-    - none of these tests have been run successfully on windows yet but
-      there's been manual testing that verified it works.
-    - wonder if any of it'll work on OS X?
+  - none of these tests have been run successfully on windows yet but
+    there's been manual testing that verified it works.
+  - wonder if any of it'll work on OS X?
 
 """
+from __future__ import annotations
 from functools import partial
 import itertools
 import platform
 import time
+from typing import (
+    TYPE_CHECKING,
+)
 
 import pytest
 from pexpect.exceptions import (
@@ -34,6 +38,9 @@ from .conftest import (
     assert_before,
 )
 
+if TYPE_CHECKING:
+    from ..conftest import PexpectSpawner
+
 # TODO: The next great debugger audit could be done by you!
 # - recurrent entry to breakpoint() from single actor *after* and an
 #   error in another task?
@@ -310,7 +317,6 @@ def test_subactor_breakpoint(
 
     assert in_prompt_msg(
         child, [
-        'MessagingError:',
         'RemoteActorError:',
          "('breakpoint_forever'",
          'bdb.BdbQuit',
@@ -528,7 +534,7 @@ def test_multi_daemon_subactors(
     # now the root actor won't clobber the bp_forever child
     # during it's first access to the debug lock, but will instead
     # wait for the lock to release, by the edge triggered
-    # ``devx._debug.Lock.no_remote_has_tty`` event before sending cancel messages
+    # ``devx.debug.Lock.no_remote_has_tty`` event before sending cancel messages
     # (via portals) to its underlings B)
 
     # at some point here there should have been some warning msg from
@@ -919,6 +925,7 @@ def test_post_mortem_api(
             "<Task 'name_error'",
             "NameError",
             "('child'",
+            'getattr(doggypants)',  # exc-LoC
         ]
     )
     if ctlc:
@@ -935,8 +942,8 @@ def test_post_mortem_api(
             "<Task '__main__.main'",
             "('root'",
             "NameError",
-            "tractor.post_mortem()",
             "src_uid=('child'",
+            "tractor.post_mortem()",  # in `main()`-LoC
         ]
     )
     if ctlc:
@@ -954,6 +961,10 @@ def test_post_mortem_api(
             "('root'",
             "NameError",
             "src_uid=('child'",
+
+            # raising line in `main()` but from crash-handling
+            # in `tractor.open_nursery()`.
+            'async with p.open_context(name_error) as (ctx, first):',
         ]
     )
     if ctlc:
@@ -1063,6 +1074,136 @@ def test_shield_pause(
     child.expect(EOF)
 
 
+@pytest.mark.parametrize(
+    'quit_early', [False, True]
+)
+def test_ctxep_pauses_n_maybe_ipc_breaks(
+    spawn: PexpectSpawner,
+    quit_early: bool,
+):
+    '''
+    Audit generator embedded `.pause()`es from within a `@context`
+    endpoint with a chan close at the end, requiring that ctl-c is
+    mashed and zombie reaper kills sub with no hangs.
+
+    '''
+    child = spawn('subactor_bp_in_ctx')
+    child.expect(PROMPT)
+
+    # 3 iters for the `gen()` pause-points
+    for i in range(3):
+        assert_before(
+            child,
+            [
+                _pause_msg,
+                "('bp_boi'",  # actor name
+                "<Task 'just_bp'",  # task name
+            ]
+        )
+        if (
+            i == 1
+            and
+            quit_early
+        ):
+            child.sendline('q')
+            child.expect(PROMPT)
+            assert_before(
+                child,
+                ["tractor._exceptions.RemoteActorError: remote task raised a 'BdbQuit'",
+                 "bdb.BdbQuit",
+                 "('bp_boi'",
+                ]
+            )
+            child.sendline('c')
+            child.expect(EOF)
+            assert_before(
+                child,
+                ["tractor._exceptions.RemoteActorError: remote task raised a 'BdbQuit'",
+                 "bdb.BdbQuit",
+                 "('bp_boi'",
+                ]
+            )
+            break  # end-of-test
+
+        child.sendline('c')
+        try:
+            child.expect(PROMPT)
+        except TIMEOUT:
+            # no prompt since we hang due to IPC chan purposely
+            # closed so verify we see error reporting as well as
+            # a failed crash-REPL request msg and can CTL-c our way
+            # out.
+            assert_before(
+                child,
+                ['peer IPC channel closed abruptly?',
+                 'another task closed this fd',
+                 'Debug lock request was CANCELLED?',
+                 "TransportClosed: 'MsgpackUDSStream' was already closed locally ?",]
+
+                # XXX races on whether these show/hit?
+                 # 'Failed to REPl via `_pause()` You called `tractor.pause()` from an already cancelled scope!',
+                 # 'AssertionError',
+            )
+            # OSc(ancel) the hanging tree
+            do_ctlc(
+                child=child,
+                expect_prompt=False,
+            )
+            child.expect(EOF)
+            assert_before(
+                child,
+                ['KeyboardInterrupt'],
+            )
+
+
+def test_crash_handling_within_cancelled_root_actor(
+    spawn: PexpectSpawner,
+):
+    '''
+    Ensure that when only a root-actor is started via `open_root_actor()`
+    we can crash-handle in debug-mode despite self-cancellation.
+
+    More-or-less ensures we conditionally shield the pause in
+    `._root.open_root_actor()`'s `await debug._maybe_enter_pm()`
+    call.
+
+    '''
+    child = spawn('root_self_cancelled_w_error')
+    child.expect(PROMPT)
+
+    assert_before(
+        child,
+        [
+            "Actor.cancel_soon()` was called!",
+            "root cancelled",
+            _pause_msg,
+            "('root'",  # actor name
+        ]
+    )
+
+    child.sendline('c')
+    child.expect(PROMPT)
+    assert_before(
+        child,
+        [
+            _crash_msg,
+            "('root'",  # actor name
+            "AssertionError",
+            "assert 0",
+        ]
+    )
+
+    child.sendline('c')
+    child.expect(EOF)
+    assert_before(
+        child,
+        [
+            "AssertionError",
+            "assert 0",
+        ]
+    )
+
+
 # TODO: better error for "non-ideal" usage from the root actor.
 # -[ ] if called from an async scope emit a message that suggests
 #    using `await tractor.pause()` instead since it's less overhead

tests/devx/test_tooling.py

@@ -13,9 +13,16 @@ TODO:
   when debugging a problem inside the stack vs. in their app.
 
 '''
+from __future__ import annotations
+from contextlib import (
+    contextmanager as cm,
+)
 import os
 import signal
 import time
+from typing import (
+    TYPE_CHECKING,
+)
 
 from .conftest import (
     expect,
@@ -24,14 +31,19 @@ from .conftest import (
     PROMPT,
     _pause_msg,
 )
+
+import pytest
 from pexpect.exceptions import (
     # TIMEOUT,
     EOF,
 )
 
+if TYPE_CHECKING:
+    from ..conftest import PexpectSpawner
+
 
 def test_shield_pause(
-    spawn,
+    spawn: PexpectSpawner,
 ):
     '''
     Verify the `tractor.pause()/.post_mortem()` API works inside an
@@ -109,9 +121,11 @@ def test_shield_pause(
         child.pid,
         signal.SIGINT,
     )
+    from tractor._supervise import _shutdown_msg
     expect(
         child,
-        'Shutting down actor runtime',
+        # 'Shutting down actor runtime',
+        _shutdown_msg,
         timeout=6,
     )
     assert_before(
@@ -126,7 +140,7 @@ def test_shield_pause(
 
 
 def test_breakpoint_hook_restored(
-    spawn,
+    spawn: PexpectSpawner,
 ):
     '''
     Ensures our actor runtime sets a custom `breakpoint()` hook
@@ -140,16 +154,22 @@ def test_breakpoint_hook_restored(
     child = spawn('restore_builtin_breakpoint')
 
     child.expect(PROMPT)
-    assert_before(
-        child,
-        [
-            _pause_msg,
-            "<Task '__main__.main'",
-            "('root'",
-            "first bp, tractor hook set",
-        ]
-    )
-    child.sendline('c')
+    try:
+        assert_before(
+            child,
+            [
+                _pause_msg,
+                "<Task '__main__.main'",
+                "('root'",
+                "first bp, tractor hook set",
+            ]
+        )
+    # XXX if the above raises `AssertionError`, without sending
+    # the final 'continue' cmd to the REPL-active sub-process,
+    # we'll hang waiting for that pexpect instance to terminate..
+    finally:
+        child.sendline('c')
+
     child.expect(PROMPT)
     assert_before(
         child,
@@ -170,3 +190,117 @@ def test_breakpoint_hook_restored(
     )
     child.sendline('c')
     child.expect(EOF)
+
+
+
+_to_raise = Exception('Triggering a crash')
+
+
+@pytest.mark.parametrize(
+    'to_raise',
+    [
+        None,
+        _to_raise,
+        RuntimeError('Never crash handle this!'),
+    ],
+)
+@pytest.mark.parametrize(
+    'raise_on_exit',
+    [
+        True,
+        [type(_to_raise)],
+        False,
+    ]
+)
+def test_crash_handler_cms(
+    debug_mode: bool,
+    to_raise: Exception,
+    raise_on_exit: bool|list[Exception],
+):
+    '''
+    Verify the `.devx.open_crash_handler()` API(s) by also
+    (conveniently enough) tesing its `repl_fixture: ContextManager`
+    param support which for this suite allows use to avoid use of
+    a `pexpect`-style-test since we use the fixture to avoid actually
+    entering `PdbpREPL.iteract()` :smirk:
+
+    '''
+    import tractor
+    # import trio
+
+    # state flags
+    repl_acquired: bool = False
+    repl_released: bool = False
+
+    @cm
+    def block_repl_ux(
+        repl: tractor.devx.debug.PdbREPL,
+        maybe_bxerr: (
+            tractor.devx._debug.BoxedMaybeException
+            |None
+        ) = None,
+        enter_repl: bool = True,
+
+    ) -> bool:
+        '''
+        Set pre/post-REPL state vars and bypass actual conole
+        interaction.
+
+        '''
+        nonlocal repl_acquired, repl_released
+
+        # task: trio.Task = trio.lowlevel.current_task()
+        # print(f'pre-REPL active_task={task.name}')
+
+        print('pre-REPL')
+        repl_acquired = True
+        yield False  # never actually .interact()
+        print('post-REPL')
+        repl_released = True
+
+    try:
+        # TODO, with runtime's `debug_mode` setting
+        # -[ ] need to open runtime tho obvi..
+        #
+        # with tractor.devx.maybe_open_crash_handler(
+        #     pdb=True,
+
+        with tractor.devx.open_crash_handler(
+            raise_on_exit=raise_on_exit,
+            repl_fixture=block_repl_ux
+        ) as bxerr:
+            if to_raise is not None:
+                raise to_raise
+
+    except Exception as _exc:
+        exc = _exc
+        if (
+            raise_on_exit is True
+            or
+            type(to_raise) in raise_on_exit
+        ):
+            assert (
+                exc
+                is
+                to_raise
+                is
+                bxerr.value
+            )
+
+        else:
+            raise
+    else:
+        assert (
+            to_raise is None
+            or
+            not raise_on_exit
+            or
+            type(to_raise) not in raise_on_exit
+        )
+        assert bxerr.value is to_raise
+
+    assert bxerr.raise_on_exit == raise_on_exit
+
+    if to_raise is not None:
+        assert repl_acquired
+        assert repl_released

tests/ipc/test_each_tpt.py

@@ -0,0 +1,114 @@
+'''
+Unit-ish tests for specific IPC transport protocol backends.
+
+'''
+from __future__ import annotations
+from pathlib import Path
+
+import pytest
+import trio
+import tractor
+from tractor import (
+    Actor,
+    _state,
+    _addr,
+)
+
+
+@pytest.fixture
+def bindspace_dir_str() -> str:
+
+    rt_dir: Path = tractor._state.get_rt_dir()
+    bs_dir: Path = rt_dir / 'doggy'
+    bs_dir_str: str = str(bs_dir)
+    assert not bs_dir.is_dir()
+
+    yield bs_dir_str
+
+    # delete it on suite teardown.
+    # ?TODO? should we support this internally
+    # or is leaking it ok?
+    if bs_dir.is_dir():
+        bs_dir.rmdir()
+
+
+def test_uds_bindspace_created_implicitly(
+    debug_mode: bool,
+    bindspace_dir_str: str,
+):
+    registry_addr: tuple = (
+        f'{bindspace_dir_str}',
+        'registry@doggy.sock',
+    )
+    bs_dir_str: str = registry_addr[0]
+
+    # XXX, ensure bindspace-dir DNE beforehand!
+    assert not Path(bs_dir_str).is_dir()
+
+    async def main():
+        async with tractor.open_nursery(
+            enable_transports=['uds'],
+            registry_addrs=[registry_addr],
+            debug_mode=debug_mode,
+        ) as _an:
+
+            # XXX MUST be created implicitly by
+            # `.ipc._uds.start_listener()`!
+            assert Path(bs_dir_str).is_dir()
+
+            root: Actor = tractor.current_actor()
+            assert root.is_registrar
+
+            assert registry_addr in root.reg_addrs
+            assert (
+                registry_addr
+                in
+                _state._runtime_vars['_registry_addrs']
+            )
+            assert (
+                _addr.wrap_address(registry_addr)
+                in
+                root.registry_addrs
+            )
+
+    trio.run(main)
+
+
+def test_uds_double_listen_raises_connerr(
+    debug_mode: bool,
+    bindspace_dir_str: str,
+):
+    registry_addr: tuple = (
+        f'{bindspace_dir_str}',
+        'registry@doggy.sock',
+    )
+
+    async def main():
+        async with tractor.open_nursery(
+            enable_transports=['uds'],
+            registry_addrs=[registry_addr],
+            debug_mode=debug_mode,
+        ) as _an:
+
+            # runtime up
+            root: Actor = tractor.current_actor()
+
+            from tractor.ipc._uds import (
+                start_listener,
+                UDSAddress,
+            )
+            ya_bound_addr: UDSAddress = root.registry_addrs[0]
+            try:
+                await start_listener(
+                    addr=ya_bound_addr,
+                )
+            except ConnectionError as connerr:
+                assert type(src_exc := connerr.__context__) is OSError
+                assert 'Address already in use' in src_exc.args
+                # complete, exit test.
+
+            else:
+                pytest.fail('It dint raise a connerr !?')
+
+
+    trio.run(main)

tests/test_advanced_streaming.py

@@ -313,9 +313,8 @@ async def inf_streamer(
         # `trio.EndOfChannel` doesn't propagate directly to the above
         # .open_stream() parent, resulting in it also raising instead
         # of gracefully absorbing as normal.. so how to handle?
-        trio.open_nursery(
-            strict_exception_groups=False,
-        ) as tn,
+        tractor.trionics.collapse_eg(),
+        trio.open_nursery() as tn,
     ):
         async def close_stream_on_sentinel():
             async for msg in stream:

tests/test_cancellation.py

@@ -236,7 +236,10 @@ async def stream_forever():
 async def test_cancel_infinite_streamer(start_method):
 
     # stream for at most 1 seconds
-    with trio.move_on_after(1) as cancel_scope:
+    with (
+        trio.fail_after(4),
+        trio.move_on_after(1) as cancel_scope
+    ):
         async with tractor.open_nursery() as n:
             portal = await n.start_actor(
                 'donny',
@@ -284,20 +287,32 @@ async def test_cancel_infinite_streamer(start_method):
     ],
 )
 @tractor_test
-async def test_some_cancels_all(num_actors_and_errs, start_method, loglevel):
-    """Verify a subset of failed subactors causes all others in
+async def test_some_cancels_all(
+    num_actors_and_errs: tuple,
+    start_method: str,
+    loglevel: str,
+):
+    '''
+    Verify a subset of failed subactors causes all others in
     the nursery to be cancelled just like the strategy in trio.
 
     This is the first and only supervisory strategy at the moment.
-    """
-    num_actors, first_err, err_type, ria_func, da_func = num_actors_and_errs
+
+    '''
+    (
+        num_actors,
+        first_err,
+        err_type,
+        ria_func,
+        da_func,
+    ) = num_actors_and_errs
     try:
-        async with tractor.open_nursery() as n:
+        async with tractor.open_nursery() as an:
 
             # spawn the same number of deamon actors which should be cancelled
             dactor_portals = []
             for i in range(num_actors):
-                dactor_portals.append(await n.start_actor(
+                dactor_portals.append(await an.start_actor(
                     f'deamon_{i}',
                     enable_modules=[__name__],
                 ))
@@ -307,7 +322,7 @@ async def test_some_cancels_all(num_actors_and_errs, start_method, loglevel):
             for i in range(num_actors):
                 # start actor(s) that will fail immediately
                 riactor_portals.append(
-                    await n.run_in_actor(
+                    await an.run_in_actor(
                         func,
                         name=f'actor_{i}',
                         **kwargs
@@ -337,7 +352,8 @@ async def test_some_cancels_all(num_actors_and_errs, start_method, loglevel):
 
         # should error here with a ``RemoteActorError`` or ``MultiError``
 
-    except first_err as err:
+    except first_err as _err:
+        err = _err
         if isinstance(err, BaseExceptionGroup):
             assert len(err.exceptions) == num_actors
             for exc in err.exceptions:
@@ -348,8 +364,8 @@ async def test_some_cancels_all(num_actors_and_errs, start_method, loglevel):
         elif isinstance(err, tractor.RemoteActorError):
             assert err.boxed_type == err_type
 
-        assert n.cancelled is True
-        assert not n._children
+        assert an.cancelled is True
+        assert not an._children
     else:
         pytest.fail("Should have gotten a remote assertion error?")
 
@@ -519,10 +535,15 @@ def test_cancel_via_SIGINT_other_task(
     async def main():
         # should never timeout since SIGINT should cancel the current program
         with trio.fail_after(timeout):
-            async with trio.open_nursery(
-                strict_exception_groups=False,
-            ) as n:
-                await n.start(spawn_and_sleep_forever)
+            async with (
+
+                # XXX ?TODO? why no work!?
+                # tractor.trionics.collapse_eg(),
+                trio.open_nursery(
+                    strict_exception_groups=False,
+                ) as tn,
+            ):
+                await tn.start(spawn_and_sleep_forever)
                 if 'mp' in spawn_backend:
                     time.sleep(0.1)
                 os.kill(pid, signal.SIGINT)
@@ -533,38 +554,123 @@ def test_cancel_via_SIGINT_other_task(
 
 async def spin_for(period=3):
     "Sync sleep."
+    print(f'sync sleeping in sub-sub for {period}\n')
     time.sleep(period)
 
 
-async def spawn():
-    async with tractor.open_nursery() as tn:
-        await tn.run_in_actor(
+async def spawn_sub_with_sync_blocking_task():
+    async with tractor.open_nursery() as an:
+        print('starting sync blocking subactor..\n')
+        await an.run_in_actor(
             spin_for,
             name='sleeper',
         )
+        print('exiting first subactor layer..\n')
 
 
+@pytest.mark.parametrize(
+    'man_cancel_outer',
+    [
+        False,  # passes if delay != 2
+
+        # always causes an unexpected eg-w-embedded-assert-err?
+        pytest.param(True,
+             marks=pytest.mark.xfail(
+                 reason=(
+                    'always causes an unexpected eg-w-embedded-assert-err?'
+                )
+            ),
+        ),
+    ],
+)
 @no_windows
 def test_cancel_while_childs_child_in_sync_sleep(
-    loglevel,
-    start_method,
-    spawn_backend,
+    loglevel: str,
+    start_method: str,
+    spawn_backend: str,
+    debug_mode: bool,
+    reg_addr: tuple,
+    man_cancel_outer: bool,
 ):
-    """Verify that a child cancelled while executing sync code is torn
+    '''
+    Verify that a child cancelled while executing sync code is torn
     down even when that cancellation is triggered by the parent
     2 nurseries "up".
-    """
+
+    Though the grandchild should stay blocking its actor runtime, its
+    parent should issue a "zombie reaper" to hard kill it after
+    sufficient timeout.
+
+    '''
     if start_method == 'forkserver':
         pytest.skip("Forksever sux hard at resuming from sync sleep...")
 
     async def main():
-        with trio.fail_after(2):
-            async with tractor.open_nursery() as tn:
-                await tn.run_in_actor(
-                    spawn,
-                    name='spawn',
+        #
+        # XXX BIG TODO NOTE XXX
+        #
+        # it seems there's a strange race that can happen
+        # where where the fail-after will trigger outer scope
+        # .cancel() which then causes the inner scope to raise,
+        #
+        # BaseExceptionGroup('Exceptions from Trio nursery', [
+        #   BaseExceptionGroup('Exceptions from Trio nursery',
+        #   [
+        #       Cancelled(),
+        #       Cancelled(),
+        #   ]
+        #   ),
+        #   AssertionError('assert 0')
+        # ])
+        #
+        # WHY THIS DOESN'T MAKE SENSE:
+        # ---------------------------
+        # - it should raise too-slow-error when too slow..
+        #  * verified that using simple-cs and manually cancelling
+        #    you get same outcome -> indicates that the fail-after
+        #    can have its TooSlowError overriden!
+        #  |_ to check this it's easy, simplly decrease the timeout
+        #     as per the var below.
+        #
+        # - when using the manual simple-cs the outcome is different
+        #   DESPITE the `assert 0` which means regardless of the
+        #   inner scope effectively failing in the same way, the
+        #   bubbling up **is NOT the same**.
+        #
+        # delays trigger diff outcomes..
+        # ---------------------------
+        # as seen by uncommenting various lines below there is from
+        # my POV an unexpected outcome due to the delay=2 case.
+        #
+        # delay = 1  # no AssertionError in eg, TooSlowError raised.
+        # delay = 2  # is AssertionError in eg AND no TooSlowError !?
+        delay = 4  # is AssertionError in eg AND no _cs cancellation.
+
+        with trio.fail_after(delay) as _cs:
+        # with trio.CancelScope() as cs:
+        # ^XXX^ can be used instead to see same outcome.
+
+            async with (
+                # tractor.trionics.collapse_eg(),  # doesn't help
+                tractor.open_nursery(
+                    hide_tb=False,
+                    debug_mode=debug_mode,
+                    registry_addrs=[reg_addr],
+                ) as an,
+            ):
+                await an.run_in_actor(
+                    spawn_sub_with_sync_blocking_task,
+                    name='sync_blocking_sub',
                 )
                 await trio.sleep(1)
+
+                if man_cancel_outer:
+                    print('Cancelling manually in root')
+                    _cs.cancel()
+
+                # trigger exc-srced taskc down
+                # the actor tree.
+                print('RAISING IN ROOT')
                 assert 0
 
     with pytest.raises(AssertionError):

tests/test_child_manages_service_nursery.py

@@ -117,9 +117,10 @@ async def open_actor_local_nursery(
     ctx: tractor.Context,
 ):
     global _nursery
-    async with trio.open_nursery(
-        strict_exception_groups=False,
-    ) as tn:
+    async with (
+        tractor.trionics.collapse_eg(),
+        trio.open_nursery() as tn
+    ):
         _nursery = tn
         await ctx.started()
         await trio.sleep(10)

tests/test_clustering.py

@@ -13,26 +13,24 @@ MESSAGE = 'tractoring at full speed'
 def test_empty_mngrs_input_raises() -> None:
 
     async def main():
-        with trio.fail_after(1):
+        with trio.fail_after(3):
             async with (
                 open_actor_cluster(
                     modules=[__name__],
 
                     # NOTE: ensure we can passthrough runtime opts
-                    loglevel='info',
-                    # debug_mode=True,
+                    loglevel='cancel',
+                    debug_mode=False,
 
                 ) as portals,
 
-                gather_contexts(
-                    # NOTE: it's the use of inline-generator syntax
-                    # here that causes the empty input.
-                    mngrs=(
-                        p.open_context(worker) for p in portals.values()
-                    ),
-                ),
+                gather_contexts(mngrs=()),
             ):
-                assert 0
+                # should fail before this?
+                assert portals
+
+                # test should fail if we mk it here!
+                assert 0, 'Should have raised val-err !?'
 
     with pytest.raises(ValueError):
         trio.run(main)

tests/test_context_stream_semantics.py

@@ -252,7 +252,7 @@ def test_simple_context(
             pass
         except BaseExceptionGroup as beg:
             # XXX: on windows it seems we may have to expect the group error
-            from tractor._exceptions import is_multi_cancelled
+            from tractor.trionics import is_multi_cancelled
             assert is_multi_cancelled(beg)
     else:
         trio.run(main)

tests/test_discovery.py

@@ -11,6 +11,7 @@ import psutil
 import pytest
 import subprocess
 import tractor
+from tractor.trionics import collapse_eg
 from tractor._testing import tractor_test
 import trio
 
@@ -193,10 +194,10 @@ async def spawn_and_check_registry(
 
             try:
                 async with tractor.open_nursery() as an:
-                    async with trio.open_nursery(
-                        strict_exception_groups=False,
-                    ) as trion:
-
+                    async with (
+                        collapse_eg(),
+                        trio.open_nursery() as trion,
+                    ):
                         portals = {}
                         for i in range(3):
                             name = f'a{i}'
@@ -338,11 +339,12 @@ async def close_chans_before_nursery(
                         async with portal2.open_stream_from(
                             stream_forever
                         ) as agen2:
-                            async with trio.open_nursery(
-                                strict_exception_groups=False,
-                            ) as n:
-                                n.start_soon(streamer, agen1)
-                                n.start_soon(cancel, use_signal, .5)
+                            async with (
+                                collapse_eg(),
+                                trio.open_nursery() as tn,
+                            ):
+                                tn.start_soon(streamer, agen1)
+                                tn.start_soon(cancel, use_signal, .5)
                                 try:
                                     await streamer(agen2)
                                 finally:

tests/test_docs_examples.py

@@ -95,6 +95,7 @@ def run_example_in_subproc(
             and 'integration' not in p[0]
             and 'advanced_faults' not in p[0]
             and 'multihost' not in p[0]
+            and 'trio' not in p[0]
         )
     ],
     ids=lambda t: t[1],

tests/test_infected_asyncio.py

@@ -234,10 +234,8 @@ async def trio_ctx(
     with trio.fail_after(1 + delay):
         try:
             async with (
-                trio.open_nursery(
-                    # TODO, for new `trio` / py3.13
-                    # strict_exception_groups=False,
-                ) as tn,
+                tractor.trionics.collapse_eg(),
+                trio.open_nursery() as tn,
                 tractor.to_asyncio.open_channel_from(
                     sleep_and_err,
                 ) as (first, chan),
@@ -573,14 +571,16 @@ def test_basic_interloop_channel_stream(
     fan_out: bool,
 ):
     async def main():
-        async with tractor.open_nursery() as an:
-            portal = await an.run_in_actor(
-                stream_from_aio,
-                infect_asyncio=True,
-                fan_out=fan_out,
-            )
-            # should raise RAE diectly
-            await portal.result()
+        # TODO, figure out min timeout here!
+        with trio.fail_after(6):
+            async with tractor.open_nursery() as an:
+                portal = await an.run_in_actor(
+                    stream_from_aio,
+                    infect_asyncio=True,
+                    fan_out=fan_out,
+                )
+                # should raise RAE diectly
+                await portal.result()
 
     trio.run(main)
 
@@ -889,7 +889,7 @@ async def manage_file(
 
         # NOTE: turns out you don't even need to sched an aio task
         # since the original issue, even though seemingly was due to
-        # the guest-run being abandoned + a `._debug.pause()` inside
+        # the guest-run being abandoned + a `.debug.pause()` inside
         # `._runtime._async_main()` (which was originally trying to
         # debug the `.lifetime_stack` not closing), IS NOT actually
         # the core issue?
@@ -1088,6 +1088,108 @@ def test_sigint_closes_lifetime_stack(
     trio.run(main)
 
 
+
+# ?TODO asyncio.Task fn-deco?
+# -[ ] do sig checkingat import time like @context?
+# -[ ] maybe name it @aio_task ??
+# -[ ] chan: to_asyncio.InterloopChannel ??
+async def raise_before_started(
+    # from_trio: asyncio.Queue,
+    # to_trio: trio.abc.SendChannel,
+    chan: to_asyncio.LinkedTaskChannel,
+
+) -> None:
+    '''
+    `asyncio.Task` entry point which RTEs before calling
+    `to_trio.send_nowait()`.
+
+    '''
+    await asyncio.sleep(0.2)
+    raise RuntimeError('Some shite went wrong before `.send_nowait()`!!')
+
+    # to_trio.send_nowait('Uhh we shouldve RTE-d ^^ ??')
+    chan.started_nowait('Uhh we shouldve RTE-d ^^ ??')
+    await asyncio.sleep(float('inf'))
+
+
+@tractor.context
+async def caching_ep(
+    ctx: tractor.Context,
+):
+
+    log = tractor.log.get_logger('caching_ep')
+    log.info('syncing via `ctx.started()`')
+    await ctx.started()
+
+    # XXX, allocate the `open_channel_from()` inside
+    # a `.trionics.maybe_open_context()`.
+    chan: to_asyncio.LinkedTaskChannel
+    async with (
+        tractor.trionics.maybe_open_context(
+            acm_func=tractor.to_asyncio.open_channel_from,
+            kwargs={
+                'target': raise_before_started,
+                # ^XXX, kwarg to `open_channel_from()`
+            },
+
+            # lock around current actor task access
+            key=tractor.current_actor().uid,
+
+        ) as (cache_hit, (clients, chan)),
+    ):
+        if cache_hit:
+            log.error(
+                'Re-using cached `.open_from_channel()` call!\n'
+            )
+
+        else:
+            log.info(
+                'Allocating SHOULD-FAIL `.open_from_channel()`\n'
+            )
+
+        await trio.sleep_forever()
+
+
+def test_aio_side_raises_before_started(
+    reg_addr: tuple[str, int],
+    debug_mode: bool,
+    loglevel: str,
+):
+    '''
+    Simulates connection-err from `piker.brokers.ib.api`..
+
+    Ensure any error raised by child-`asyncio.Task` BEFORE
+    `chan.started()`
+
+    '''
+    # delay = 999 if debug_mode else 1
+    async def main():
+        with trio.fail_after(3):
+            an: tractor.ActorNursery
+            async with tractor.open_nursery(
+                debug_mode=debug_mode,
+                loglevel=loglevel,
+            ) as an:
+                p: tractor.Portal = await an.start_actor(
+                    'lchan_cacher_that_raises_fast',
+                    enable_modules=[__name__],
+                    infect_asyncio=True,
+                )
+                async with p.open_context(
+                    caching_ep,
+                ) as (ctx, first):
+                    assert not first
+
+    with pytest.raises(
+        expected_exception=(RemoteActorError),
+    ) as excinfo:
+        trio.run(main)
+
+    # ensure `asyncio.Task` exception is bubbled
+    # allll the way erp!!
+    rae = excinfo.value
+    assert rae.boxed_type is RuntimeError
+
 # TODO: debug_mode tests once we get support for `asyncio`!
 #
 # -[ ] need tests to wrap both scripts:
@@ -1101,7 +1203,7 @@ def test_sigint_closes_lifetime_stack(
 #    => completed using `.bestow_portal(task)` inside
 #     `.to_asyncio._run_asyncio_task()` right?
 #   -[ ] translation func to get from `asyncio` task calling to 
-#     `._debug.wait_for_parent_stdin_hijack()` which does root
+#     `.debug.wait_for_parent_stdin_hijack()` which does root
 #     call to do TTY locking.
 #
 def test_sync_breakpoint():

tests/test_inter_peer_cancellation.py

@@ -24,14 +24,10 @@ from tractor._testing import (
 )
 
 # XXX TODO cases:
-# - [ ] peer cancelled itself - so other peers should
-#   get errors reflecting that the peer was itself the .canceller?
-
 # - [x] WE cancelled the peer and thus should not see any raised
 #   `ContextCancelled` as it should be reaped silently?
 #   => pretty sure `test_context_stream_semantics::test_caller_cancels()`
 #      already covers this case?
-
 # - [x] INTER-PEER: some arbitrary remote peer cancels via
 #   Portal.cancel_actor().
 #   => all other connected peers should get that cancel requesting peer's
@@ -44,16 +40,6 @@ from tractor._testing import (
 #   that also spawned a remote task task in that same peer-parent.
 
 
-# def test_self_cancel():
-#     '''
-#     2 cases:
-#     - calls `Actor.cancel()` locally in some task
-#     - calls LocalPortal.cancel_actor()` ?
-
-#     '''
-#     ...
-
-
 @tractor.context
 async def open_stream_then_sleep_forever(
     ctx: Context,
@@ -410,7 +396,6 @@ def test_peer_canceller(
     '''
     async def main():
         async with tractor.open_nursery(
-            # NOTE: to halt the peer tasks on ctxc, uncomment this.
             debug_mode=debug_mode,
         ) as an:
             canceller: Portal = await an.start_actor(
@@ -807,7 +792,7 @@ async def basic_echo_server(
     ctx: Context,
     peer_name: str = 'wittle_bruv',
 
-    err_after: int|None = None,
+    err_after_imsg: int|None = None,
 
 ) -> None:
     '''
@@ -836,8 +821,9 @@ async def basic_echo_server(
             await ipc.send(resp)
 
             if (
-                err_after
-                and i > err_after
+                err_after_imsg
+                and
+                i > err_after_imsg
             ):
                 raise RuntimeError(
                     f'Simulated error in `{peer_name}`'
@@ -979,7 +965,8 @@ async def tell_little_bro(
     actor_name: str,
 
     caller: str = '',
-    err_after: int|None = None,
+    err_after: float|None = None,
+    rng_seed: int = 50,
 ):
     # contact target actor, do a stream dialog.
     async with (
@@ -990,14 +977,18 @@ async def tell_little_bro(
             basic_echo_server,
 
             # XXX proxy any delayed err condition
-            err_after=err_after,
+            err_after_imsg=(
+                err_after * rng_seed
+                if err_after is not None
+                else None
+            ),
         ) as (sub_ctx, first),
 
         sub_ctx.open_stream() as echo_ipc,
     ):
         actor: Actor = current_actor()
         uid: tuple = actor.uid
-        for i in range(100):
+        for i in range(rng_seed):
             msg: tuple = (
                 uid,
                 i,
@@ -1022,13 +1013,13 @@ async def tell_little_bro(
 )
 @pytest.mark.parametrize(
     'raise_sub_spawn_error_after',
-    [None, 50],
+    [None, 0.5],
 )
 def test_peer_spawns_and_cancels_service_subactor(
     debug_mode: bool,
     raise_client_error: str,
     reg_addr: tuple[str, int],
-    raise_sub_spawn_error_after: int|None,
+    raise_sub_spawn_error_after: float|None,
 ):
     # NOTE: this tests for the modden `mod wks open piker` bug
     # discovered as part of implementing workspace ctx
@@ -1042,6 +1033,7 @@ def test_peer_spawns_and_cancels_service_subactor(
     #   and the server's spawned child should cancel and terminate!
     peer_name: str = 'little_bro'
 
+
     def check_inner_rte(rae: RemoteActorError):
         '''
         Validate the little_bro's relayed inception!
@@ -1135,8 +1127,7 @@ def test_peer_spawns_and_cancels_service_subactor(
                         )
 
                     try:
-                        res = await client_ctx.result(hide_tb=False)
-
+                        res = await client_ctx.wait_for_result(hide_tb=False)
                         # in remote (relayed inception) error
                         # case, we should error on the line above!
                         if raise_sub_spawn_error_after:
@@ -1147,6 +1138,23 @@ def test_peer_spawns_and_cancels_service_subactor(
                         assert isinstance(res, ContextCancelled)
                         assert client_ctx.cancel_acked
                         assert res.canceller == root.uid
+                        assert not raise_sub_spawn_error_after
+
+                        # cancelling the spawner sub should
+                        # transitively cancel it's sub, the little
+                        # bruv.
+                        print('root cancelling server/client sub-actors')
+                        await spawn_ctx.cancel()
+                        async with tractor.find_actor(
+                            name=peer_name,
+                        ) as sub:
+                            assert not sub
+
+                    # XXX, only for tracing
+                    # except BaseException as _berr:
+                    #     berr = _berr
+                    #     await tractor.pause(shield=True)
+                    #     raise berr
 
                     except RemoteActorError as rae:
                         _err = rae
@@ -1175,19 +1183,8 @@ def test_peer_spawns_and_cancels_service_subactor(
                         raise
                         # await tractor.pause()
 
-                    else:
-                        assert not raise_sub_spawn_error_after
-
-                        # cancelling the spawner sub should
-                        # transitively cancel it's sub, the little
-                        # bruv.
-                        print('root cancelling server/client sub-actors')
-                        await spawn_ctx.cancel()
-                        async with tractor.find_actor(
-                            name=peer_name,
-                        ) as sub:
-                            assert not sub
 
+                    # await tractor.pause()
                     # await server.cancel_actor()
 
             except RemoteActorError as rae:
@@ -1200,7 +1197,7 @@ def test_peer_spawns_and_cancels_service_subactor(
 
             # since we called `.cancel_actor()`, `.cancel_ack`
             # will not be set on the ctx bc `ctx.cancel()` was not
-            # called directly fot this confext.
+            # called directly for this confext.
             except ContextCancelled as ctxc:
                 _ctxc = ctxc
                 print(
@@ -1240,12 +1237,19 @@ def test_peer_spawns_and_cancels_service_subactor(
 
                 # assert spawn_ctx.cancelled_caught
 
+    async def _main():
+        with trio.fail_after(
+            3 if not debug_mode
+            else 999
+        ):
+            await main()
+
     if raise_sub_spawn_error_after:
         with pytest.raises(RemoteActorError) as excinfo:
-            trio.run(main)
+            trio.run(_main)
 
         rae: RemoteActorError = excinfo.value
         check_inner_rte(rae)
 
     else:
-        trio.run(main)
+        trio.run(_main)

tests/test_legacy_one_way_streaming.py

@@ -235,10 +235,16 @@ async def cancel_after(wait, reg_addr):
 
 
 @pytest.fixture(scope='module')
-def time_quad_ex(reg_addr, ci_env, spawn_backend):
+def time_quad_ex(
+    reg_addr: tuple,
+    ci_env: bool,
+    spawn_backend: str,
+):
     if spawn_backend == 'mp':
-        """no idea but the  mp *nix runs are flaking out here often...
-        """
+        '''
+        no idea but the  mp *nix runs are flaking out here often...
+
+        '''
         pytest.skip("Test is too flaky on mp in CI")
 
     timeout = 7 if platform.system() in ('Windows', 'Darwin') else 4
@@ -249,12 +255,24 @@ def time_quad_ex(reg_addr, ci_env, spawn_backend):
     return results, diff
 
 
-def test_a_quadruple_example(time_quad_ex, ci_env, spawn_backend):
-    """This also serves as a kind of "we'd like to be this fast test"."""
+def test_a_quadruple_example(
+    time_quad_ex: tuple,
+    ci_env: bool,
+    spawn_backend: str,
+):
+    '''
+    This also serves as a kind of "we'd like to be this fast test".
 
+    '''
     results, diff = time_quad_ex
     assert results
-    this_fast = 6 if platform.system() in ('Windows', 'Darwin') else 3
+    this_fast = (
+        6 if platform.system() in (
+            'Windows',
+            'Darwin',
+        )
+        else 3
+    )
     assert diff < this_fast
 
 

tests/test_oob_cancellation.py

@@ -0,0 +1,239 @@
+'''
+Define the details of inter-actor "out-of-band" (OoB) cancel
+semantics, that is how cancellation works when a cancel request comes
+from the different concurrency (primitive's) "layer" then where the
+eventual `trio.Task` actually raises a signal.
+
+'''
+from functools import partial
+# from contextlib import asynccontextmanager as acm
+# import itertools
+
+import pytest
+import trio
+import tractor
+from tractor import (  # typing
+    ActorNursery,
+    Portal,
+    Context,
+    # ContextCancelled,
+    # RemoteActorError,
+)
+# from tractor._testing import (
+#     tractor_test,
+#     expect_ctxc,
+# )
+
+# XXX TODO cases:
+# - [ ] peer cancelled itself - so other peers should
+#   get errors reflecting that the peer was itself the .canceller?
+
+# def test_self_cancel():
+#     '''
+#     2 cases:
+#     - calls `Actor.cancel()` locally in some task
+#     - calls LocalPortal.cancel_actor()` ?
+#
+# things to ensure!
+# -[ ] the ctxc raised in a child should ideally show the tb of the
+#     underlying `Cancelled` checkpoint, i.e.
+#     `raise scope_error from ctxc`?
+#
+# -[ ] a self-cancelled context, if not allowed to block on
+#     `ctx.result()` at some point will hang since the `ctx._scope`
+#     is never `.cancel_called`; cases for this include,
+#     - an `open_ctx()` which never starteds before being OoB actor
+#       cancelled.
+#       |_ parent task will be blocked in `.open_context()` for the
+#         `Started` msg, and when the OoB ctxc arrives `ctx._scope`
+#         will never have been signalled..
+
+#     '''
+#     ...
+
+# TODO, sanity test against the case in `/examples/trio/lockacquire_not_unmasked.py`
+# but with the `Lock.acquire()` from a `@context` to ensure the
+# implicit ignore-case-non-unmasking.
+#
+# @tractor.context
+# async def acquire_actor_global_lock(
+#     ctx: tractor.Context,
+#     ignore_special_cases: bool,
+# ):
+
+#     async with maybe_unmask_excs(
+#         ignore_special_cases=ignore_special_cases,
+#     ):
+#         await ctx.started('locked')
+
+#     # block til cancelled
+#     await trio.sleep_forever()
+
+
+@tractor.context
+async def sleep_forever(
+    ctx: tractor.Context,
+    # ignore_special_cases: bool,
+    do_started: bool,
+):
+
+    # async with maybe_unmask_excs(
+    #     ignore_special_cases=ignore_special_cases,
+    # ):
+    #     await ctx.started('locked')
+    if do_started:
+        await ctx.started()
+
+    # block til cancelled
+    print('sleepin on child-side..')
+    await trio.sleep_forever()
+
+
+@pytest.mark.parametrize(
+    'cancel_ctx',
+    [True, False],
+)
+def test_cancel_ctx_with_parent_side_entered_in_bg_task(
+    debug_mode: bool,
+    loglevel: str,
+    cancel_ctx: bool,
+):
+    '''
+    The most "basic" out-of-band-task self-cancellation case where
+    `Portal.open_context()` is entered in a bg task and the
+    parent-task (of the containing nursery) calls `Context.cancel()`
+    without the child knowing; the `Context._scope` should be
+    `.cancel_called` when the IPC ctx's child-side relays
+    a `ContextCancelled` with a `.canceller` set to the parent
+    actor('s task).
+
+    '''
+    async def main():
+        with trio.fail_after(
+            2 if not debug_mode else 999,
+        ):
+            an: ActorNursery
+            async with (
+                tractor.open_nursery(
+                    debug_mode=debug_mode,
+                    loglevel='devx',
+                    enable_stack_on_sig=True,
+                ) as an,
+                trio.open_nursery() as tn,
+            ):
+                ptl: Portal = await an.start_actor(
+                    'sub',
+                    enable_modules=[__name__],
+                )
+
+                async def _open_ctx_async(
+                    do_started: bool = True,
+                    task_status=trio.TASK_STATUS_IGNORED,
+                ):
+                    # do we expect to never enter the
+                    # `.open_context()` below.
+                    if not do_started:
+                        task_status.started()
+
+                    async with ptl.open_context(
+                        sleep_forever,
+                        do_started=do_started,
+                    ) as (ctx, first):
+                        task_status.started(ctx)
+                        await trio.sleep_forever()
+
+                # XXX, this is the key OoB part!
+                #
+                # - start the `.open_context()` in a bg task which
+                #   blocks inside the embedded scope-body,
+                #
+                # -  when we call `Context.cancel()` it **is
+                #   not** from the same task which eventually runs
+                #   `.__aexit__()`,
+                #
+                # - since the bg "opener" task will be in
+                #   a `trio.sleep_forever()`, it must be interrupted
+                #   by the `ContextCancelled` delivered from the
+                #   child-side; `Context._scope: CancelScope` MUST
+                #   be `.cancel_called`!
+                #
+                print('ASYNC opening IPC context in subtask..')
+                maybe_ctx: Context|None = await tn.start(partial(
+                    _open_ctx_async,
+                ))
+
+                if (
+                    maybe_ctx
+                    and
+                    cancel_ctx
+                ):
+                    print('cancelling first IPC ctx!')
+                    await maybe_ctx.cancel()
+
+                # XXX, note that despite `maybe_context.cancel()`
+                # being called above, it's the parent (bg) task
+                # which was originally never interrupted in
+                # the `ctx._scope` body due to missing case logic in
+                # `ctx._maybe_cancel_and_set_remote_error()`.
+                #
+                # It didn't matter that the subactor process was
+                # already terminated and reaped, nothing was
+                # cancelling the ctx-parent task's scope!
+                #
+                print('cancelling subactor!')
+                await ptl.cancel_actor()
+
+                if maybe_ctx:
+                    try:
+                        await maybe_ctx.wait_for_result()
+                    except tractor.ContextCancelled as ctxc:
+                        assert not cancel_ctx
+                        assert (
+                            ctxc.canceller
+                            ==
+                            tractor.current_actor().aid.uid
+                        )
+                        # don't re-raise since it'll trigger
+                        # an EG from the above tn.
+
+    if cancel_ctx:
+        # graceful self-cancel
+        trio.run(main)
+
+    else:
+        # ctx parent task should see OoB ctxc due to
+        # `ptl.cancel_actor()`.
+        with pytest.raises(tractor.ContextCancelled) as excinfo:
+            trio.run(main)
+
+        assert 'root' in excinfo.value.canceller[0]
+
+
+# def test_parent_actor_cancels_subactor_with_gt1_ctxs_open_to_it(
+#     debug_mode: bool,
+#     loglevel: str,
+# ):
+#     '''
+#     Demos OoB cancellation from the perspective of a ctx opened with
+#     a child subactor where the parent cancels the child at the "actor
+#     layer" using `Portal.cancel_actor()` and thus the
+#     `ContextCancelled.canceller` received by the ctx's parent-side
+#     task will appear to be a "self cancellation" even though that
+#     specific task itself was not cancelled and thus
+#     `Context.cancel_called ==False`.
+#     '''
+                # TODO, do we have an existing implied ctx
+                # cancel test like this?
+                # with trio.move_on_after(0.5):# as cs:
+                #     await _open_ctx_async(
+                #         do_started=False,
+                #     )
+
+
+                # in-line ctx scope should definitely raise
+                # a ctxc with `.canceller = 'root'`
+                # async with ptl.open_context(
+                #     sleep_forever,
+                #     do_started=True,
+                # ) as pair:
+

tests/test_remote_exc_relay.py

@@ -0,0 +1,237 @@
+'''
+Special case testing for issues not (dis)covered in the primary
+`Context` related functional/scenario suites.
+
+**NOTE: this mod is a WIP** space for handling
+odd/rare/undiscovered/not-yet-revealed faults which either
+loudly (ideal case) breakl our supervision protocol
+or (worst case) result in distributed sys hangs.
+
+Suites here further try to clarify (if [partially] ill-defined) and
+verify our edge case semantics for inter-actor-relayed-exceptions
+including,
+
+- lowlevel: what remote obj-data is interchanged for IPC and what is
+  native-obj form is expected from unpacking in the the new
+  mem-domain.
+
+- which kinds of `RemoteActorError` (and its derivs) are expected by which
+  (types of) peers (parent, child, sibling, etc) with what
+  particular meta-data set such as,
+
+  - `.src_uid`: the original (maybe) peer who raised.
+  - `.relay_uid`: the next-hop-peer who sent it.
+  - `.relay_path`: the sequence of peer actor hops.
+  - `.is_inception`: a predicate that denotes multi-hop remote errors.
+
+- when should `ExceptionGroup`s be relayed from a particular
+  remote endpoint, they should never be caused by implicit `._rpc`
+  nursery machinery!
+
+- various special `trio` edge cases around its cancellation semantics
+  and how we (currently) leverage `trio.Cancelled` as a signal for
+  whether a `Context` task should raise `ContextCancelled` (ctx).
+
+'''
+import pytest
+import trio
+import tractor
+from tractor import (  # typing
+    ActorNursery,
+    Portal,
+    Context,
+    ContextCancelled,
+)
+
+
+@tractor.context
+async def sleep_n_chkpt_in_finally(
+    ctx: Context,
+    sleep_n_raise: bool,
+
+    chld_raise_delay: float,
+    chld_finally_delay: float,
+
+    rent_cancels: bool,
+    rent_ctxc_delay: float,
+
+    expect_exc: str|None = None,
+
+) -> None:
+    '''
+    Sync, open a tn, then wait for cancel, run a chkpt inside
+    the user's `finally:` teardown.
+
+    This covers a footgun case that `trio` core doesn't seem to care about
+    wherein an exc can be masked by a `trio.Cancelled` raised inside a tn emedded
+    `finally:`.
+
+    Also see `test_trioisms::test_acm_embedded_nursery_propagates_enter_err`
+    for the down and gritty details.
+
+    Since a `@context` endpoint fn can also contain code like this,
+    **and** bc we currently have no easy way other then
+    `trio.Cancelled` to signal cancellation on each side of an IPC `Context`,
+    the footgun issue can compound itself as demonstrated in this suite..
+
+    Here are some edge cases codified with our WIP "sclang" syntax
+    (note the parent(rent)/child(chld) naming here is just
+    pragmatism, generally these most of these cases can occurr
+    regardless of the distributed-task's supervision hiearchy),
+
+    - rent c)=> chld.raises-then-taskc-in-finally
+     |_ chld's body raises an `exc: BaseException`.
+      _ in its `finally:` block it runs a chkpoint
+        which raises a taskc (`trio.Cancelled`) which
+        masks `exc` instead raising taskc up to the first tn.
+      _ the embedded/chld tn captures the masking taskc and then
+        raises it up to the ._rpc-ep-tn instead of `exc`.
+      _ the rent thinks the child ctxc-ed instead of errored..
+
+    '''
+    await ctx.started()
+
+    if expect_exc:
+        expect_exc: BaseException = tractor._exceptions.get_err_type(
+            type_name=expect_exc,
+        )
+
+    berr: BaseException|None = None
+    try:
+        if not sleep_n_raise:
+            await trio.sleep_forever()
+        elif sleep_n_raise:
+
+            # XXX this sleep is less then the sleep the parent
+            # does before calling `ctx.cancel()`
+            await trio.sleep(chld_raise_delay)
+
+            # XXX this will be masked by a taskc raised in
+            # the `finally:` if this fn doesn't terminate
+            # before any ctxc-req arrives AND a checkpoint is hit
+            # in that `finally:`.
+            raise RuntimeError('my app krurshed..')
+
+    except BaseException as _berr:
+        berr = _berr
+
+        # TODO: it'd sure be nice to be able to inject our own
+        # `ContextCancelled` here instead of of `trio.Cancelled`
+        # so that our runtime can expect it and this "user code"
+        # would be able to tell the diff between a generic trio
+        # cancel and a tractor runtime-IPC cancel.
+        if expect_exc:
+            if not isinstance(
+                berr,
+                expect_exc,
+            ):
+                raise ValueError(
+                    f'Unexpected exc type ??\n'
+                    f'{berr!r}\n'
+                    f'\n'
+                    f'Expected a {expect_exc!r}\n'
+                )
+
+        raise berr
+
+    # simulate what user code might try even though
+    # it's a known boo-boo..
+    finally:
+        # maybe wait for rent ctxc to arrive
+        with trio.CancelScope(shield=True):
+            await trio.sleep(chld_finally_delay)
+
+        # !!XXX this will raise `trio.Cancelled` which
+        # will mask the RTE from above!!!
+        #
+        # YES, it's the same case as our extant
+        # `test_trioisms::test_acm_embedded_nursery_propagates_enter_err`
+        try:
+            await trio.lowlevel.checkpoint()
+        except trio.Cancelled as taskc:
+            if (scope_err := taskc.__context__):
+                print(
+                    f'XXX MASKED REMOTE ERROR XXX\n'
+                    f'ENDPOINT exception -> {scope_err!r}\n'
+                    f'will be masked by -> {taskc!r}\n'
+                )
+                # await tractor.pause(shield=True)
+
+            raise taskc
+
+
+@pytest.mark.parametrize(
+    'chld_callspec',
+    [
+        dict(
+            sleep_n_raise=None,
+            chld_raise_delay=0.1,
+            chld_finally_delay=0.1,
+            expect_exc='Cancelled',
+            rent_cancels=True,
+            rent_ctxc_delay=0.1,
+        ),
+        dict(
+            sleep_n_raise='RuntimeError',
+            chld_raise_delay=0.1,
+            chld_finally_delay=1,
+            expect_exc='RuntimeError',
+            rent_cancels=False,
+            rent_ctxc_delay=0.1,
+        ),
+    ],
+    ids=lambda item: f'chld_callspec={item!r}'
+)
+def test_unmasked_remote_exc(
+    debug_mode: bool,
+    chld_callspec: dict,
+    tpt_proto: str,
+):
+    expect_exc_str: str|None = chld_callspec['sleep_n_raise']
+    rent_ctxc_delay: float|None = chld_callspec['rent_ctxc_delay']
+    async def main():
+        an: ActorNursery
+        async with tractor.open_nursery(
+            debug_mode=debug_mode,
+            enable_transports=[tpt_proto],
+        ) as an:
+            ptl: Portal = await an.start_actor(
+                'cancellee',
+                enable_modules=[__name__],
+            )
+            ctx: Context
+            async with (
+                ptl.open_context(
+                    sleep_n_chkpt_in_finally,
+                    **chld_callspec,
+                ) as (ctx, sent),
+            ):
+                assert not sent
+                await trio.sleep(rent_ctxc_delay)
+                await ctx.cancel()
+
+                # recv error or result from chld
+                ctxc: ContextCancelled = await ctx.wait_for_result()
+                assert (
+                    ctxc is ctx.outcome
+                    and
+                    isinstance(ctxc, ContextCancelled)
+                )
+
+            # always graceful terminate the sub in non-error cases
+            await an.cancel()
+
+    if expect_exc_str:
+        expect_exc: BaseException = tractor._exceptions.get_err_type(
+            type_name=expect_exc_str,
+        )
+        with pytest.raises(
+            expected_exception=tractor.RemoteActorError,
+        ) as excinfo:
+            trio.run(main)
+
+        rae = excinfo.value
+        assert expect_exc == rae.boxed_type
+
+    else:
+        trio.run(main)

tests/test_resource_cache.py

@@ -1,5 +1,6 @@
 '''
-Async context manager cache api testing: ``trionics.maybe_open_context():``
+Suites for our `.trionics.maybe_open_context()` multi-task
+shared-cached `@acm` API.
 
 '''
 from contextlib import asynccontextmanager as acm
@@ -9,6 +10,15 @@ from typing import Awaitable
 import pytest
 import trio
 import tractor
+from tractor.trionics import (
+    maybe_open_context,
+)
+from tractor.log import (
+    get_console_log,
+    get_logger,
+)
+log = get_logger(__name__)
+
 
 
 _resource: int = 0
@@ -52,7 +62,7 @@ def test_resource_only_entered_once(key_on):
                 # different task names per task will be used
                 kwargs = {'task_name': name}
 
-            async with tractor.trionics.maybe_open_context(
+            async with maybe_open_context(
                 maybe_increment_counter,
                 kwargs=kwargs,
                 key=key,
@@ -72,11 +82,13 @@ def test_resource_only_entered_once(key_on):
         with trio.move_on_after(0.5):
             async with (
                 tractor.open_root_actor(),
-                trio.open_nursery() as n,
+                trio.open_nursery() as tn,
             ):
-
                 for i in range(10):
-                    n.start_soon(enter_cached_mngr, f'task_{i}')
+                    tn.start_soon(
+                        enter_cached_mngr,
+                        f'task_{i}',
+                    )
                     await trio.sleep(0.001)
 
     trio.run(main)
@@ -98,23 +110,34 @@ async def streamer(
 
 
 @acm
-async def open_stream() -> Awaitable[tractor.MsgStream]:
-
+async def open_stream() -> Awaitable[
+    tuple[
+        tractor.ActorNursery,
+        tractor.MsgStream,
+    ]
+]:
     try:
         async with tractor.open_nursery() as an:
             portal = await an.start_actor(
                 'streamer',
                 enable_modules=[__name__],
             )
-            async with (
-                portal.open_context(streamer) as (ctx, first),
-                ctx.open_stream() as stream,
-            ):
-                yield stream
+            try:
+                async with (
+                    portal.open_context(streamer) as (ctx, first),
+                    ctx.open_stream() as stream,
+                ):
+                    print('Entered open_stream() caller')
+                    yield an, stream
+                    print('Exited open_stream() caller')
 
-            print('Cancelling streamer')
-            await portal.cancel_actor()
-            print('Cancelled streamer')
+            finally:
+                print(
+                    'Cancelling streamer with,\n'
+                    '=> `Portal.cancel_actor()`'
+                )
+                await portal.cancel_actor()
+                print('Cancelled streamer')
 
     except Exception as err:
         print(
@@ -127,11 +150,15 @@ async def open_stream() -> Awaitable[tractor.MsgStream]:
 
 @acm
 async def maybe_open_stream(taskname: str):
-    async with tractor.trionics.maybe_open_context(
+    async with maybe_open_context(
         # NOTE: all secondary tasks should cache hit on the same key
         acm_func=open_stream,
-    ) as (cache_hit, stream):
-
+    ) as (
+        cache_hit,
+        (an, stream)
+    ):
+        # when the actor + portal + ctx + stream has already been
+        # allocated we want to just bcast to this task.
         if cache_hit:
             print(f'{taskname} loaded from cache')
 
@@ -139,10 +166,43 @@ async def maybe_open_stream(taskname: str):
             # if this feed is already allocated by the first
             # task that entereed
             async with stream.subscribe() as bstream:
-                yield bstream
+                yield an, bstream
+                print(
+                    f'cached task exited\n'
+                    f')>\n'
+                    f' |_{taskname}\n'
+                )
+
+            # we should always unreg the "cloned" bcrc for this
+            # consumer-task
+            assert id(bstream) not in bstream._state.subs
+
         else:
             # yield the actual stream
-            yield stream
+            try:
+                yield an, stream
+            finally:
+                print(
+                    f'NON-cached task exited\n'
+                    f')>\n'
+                    f' |_{taskname}\n'
+                )
+
+        first_bstream = stream._broadcaster
+        bcrx_state = first_bstream._state
+        subs: dict[int, int] = bcrx_state.subs
+        if len(subs) == 1:
+            assert id(first_bstream) in subs
+            # ^^TODO! the bcrx should always de-allocate all subs,
+            # including the implicit first one allocated on entry
+            # by the first subscribing peer task, no?
+            #
+            # -[ ] adjust `MsgStream.subscribe()` to do this mgmt!
+            #  |_ allows reverting `MsgStream.receive()` to the
+            #    non-bcaster method.
+            #  |_ we can decide whether to reset `._broadcaster`?
+            #
+            # await tractor.pause(shield=True)
 
 
 def test_open_local_sub_to_stream(
@@ -159,16 +219,24 @@ def test_open_local_sub_to_stream(
 
     if debug_mode:
         timeout = 999
+        print(f'IN debug_mode, setting large timeout={timeout!r}..')
 
     async def main():
 
         full = list(range(1000))
+        an: tractor.ActorNursery|None = None
+        num_tasks: int = 10
 
         async def get_sub_and_pull(taskname: str):
 
+            nonlocal an
+
             stream: tractor.MsgStream
             async with (
-                maybe_open_stream(taskname) as stream,
+                maybe_open_stream(taskname) as (
+                    an,
+                    stream,
+                ),
             ):
                 if '0' in taskname:
                     assert isinstance(stream, tractor.MsgStream)
@@ -180,34 +248,159 @@ def test_open_local_sub_to_stream(
 
                 first = await stream.receive()
                 print(f'{taskname} started with value {first}')
-                seq = []
+                seq: list[int] = []
                 async for msg in stream:
                     seq.append(msg)
 
                 assert set(seq).issubset(set(full))
+
+            # end of @acm block
             print(f'{taskname} finished')
 
+        root: tractor.Actor
         with trio.fail_after(timeout) as cs:
             # TODO: turns out this isn't multi-task entrant XD
             # We probably need an indepotent entry semantic?
             async with tractor.open_root_actor(
                 debug_mode=debug_mode,
-            ):
+                # maybe_enable_greenback=True,
+                #
+                # ^TODO? doesn't seem to mk breakpoint() usage work
+                # bc each bg task needs to open a portal??
+                # - [ ] we should consider making this part of
+                #      our taskman defaults?
+                #   |_see https://github.com/goodboy/tractor/pull/363
+                #
+            ) as root:
+                assert root.is_registrar
+
                 async with (
                     trio.open_nursery() as tn,
                 ):
-                    for i in range(10):
+                    for i in range(num_tasks):
                         tn.start_soon(
                             get_sub_and_pull,
                             f'task_{i}',
                         )
                         await trio.sleep(0.001)
 
-                print('all consumer tasks finished')
+                print('all consumer tasks finished!')
+
+                # ?XXX, ensure actor-nursery is shutdown or we might
+                # hang here due to a minor task deadlock/race-condition?
+                #
+                # - seems that all we need is a checkpoint to ensure
+                #   the last suspended task, which is inside
+                #   `.maybe_open_context()`, can do the
+                #   `Portal.cancel_actor()` call?
+                #
+                # - if that bg task isn't resumed, then this blocks
+                #   timeout might hit before that?
+                #
+                if root.ipc_server.has_peers():
+                    await trio.lowlevel.checkpoint()
+
+                    # alt approach, cancel the entire `an`
+                    # await tractor.pause()
+                    # await an.cancel()
+
+            # end of runtime scope
+            print('root actor terminated.')
 
         if cs.cancelled_caught:
             pytest.fail(
                 'Should NOT time out in `open_root_actor()` ?'
             )
 
+        print('exiting main.')
+
+    trio.run(main)
+
+
+
+@acm
+async def cancel_outer_cs(
+    cs: trio.CancelScope|None = None,
+    delay: float = 0,
+):
+    # on first task delay this enough to block
+    # the 2nd task but then cancel it mid sleep
+    # so that the tn.start() inside the key-err handler block
+    # is cancelled and would previously corrupt the
+    # mutext state.
+    log.info(f'task entering sleep({delay})')
+    await trio.sleep(delay)
+    if cs:
+        log.info('task calling cs.cancel()')
+        cs.cancel()
+    trio.lowlevel.checkpoint()
+    yield
+    await trio.sleep_forever()
+
+
+def test_lock_not_corrupted_on_fast_cancel(
+    debug_mode: bool,
+    loglevel: str,
+):
+    '''
+    Verify that if the caching-task (the first to enter
+    `maybe_open_context()`) is cancelled mid-cache-miss, the embedded
+    mutex can never be left in a corrupted state.
+
+    That is, the lock is always eventually released ensuring a peer
+    (cache-hitting) task will never,
+
+    - be left to inf-block/hang on the `lock.acquire()`.
+    - try to release the lock when still owned by the caching-task
+      due to it having erronously exited without calling
+      `lock.release()`.
+
+
+    '''
+    delay: float = 1.
+
+    async def use_moc(
+        cs: trio.CancelScope|None,
+        delay: float,
+    ):
+        log.info('task entering moc')
+        async with maybe_open_context(
+            cancel_outer_cs,
+            kwargs={
+                'cs': cs,
+                'delay': delay,
+            },
+        ) as (cache_hit, _null):
+            if cache_hit:
+                log.info('2nd task entered')
+            else:
+                log.info('1st task entered')
+
+            await trio.sleep_forever()
+
+    async def main():
+        with trio.fail_after(delay + 2):
+            async with (
+                tractor.open_root_actor(
+                    debug_mode=debug_mode,
+                    loglevel=loglevel,
+                ),
+                trio.open_nursery() as tn,
+            ):
+                get_console_log('info')
+                log.info('yo starting')
+                cs = tn.cancel_scope
+                tn.start_soon(
+                    use_moc,
+                    cs,
+                    delay,
+                    name='child',
+                )
+                with trio.CancelScope() as rent_cs:
+                    await use_moc(
+                        cs=rent_cs,
+                        delay=delay,
+                    )
+
+
     trio.run(main)

tests/test_root_infect_asyncio.py

@@ -147,8 +147,7 @@ def test_trio_prestarted_task_bubbles(
         await trio.sleep_forever()
 
     async def _trio_main():
-        # with trio.fail_after(2):
-        with trio.fail_after(999):
+        with trio.fail_after(2 if not debug_mode else 999):
             first: str
             chan: to_asyncio.LinkedTaskChannel
             aio_ev = asyncio.Event()
@@ -217,32 +216,25 @@ def test_trio_prestarted_task_bubbles(
                         ):
                             aio_ev.set()
 
-    with pytest.raises(
-        expected_exception=ExceptionGroup,
-    ) as excinfo:
-        tractor.to_asyncio.run_as_asyncio_guest(
-            trio_main=_trio_main,
-        )
-
-    eg = excinfo.value
-    rte_eg, rest_eg = eg.split(RuntimeError)
-
     # ensure the trio-task's error bubbled despite the aio-side
     # having (maybe) errored first.
     if aio_err_trigger in (
         'after_trio_task_starts',
         'after_start_point',
     ):
-        assert len(errs := rest_eg.exceptions) == 1
-        typerr = errs[0]
-        assert (
-            type(typerr) is TypeError
-            and
-            'trio-side' in typerr.args
-        )
+        patt: str = 'trio-side'
+        expect_exc = TypeError
 
     # when aio errors BEFORE (last) trio task is scheduled, we should
     # never see anythinb but the aio-side.
     else:
-        assert len(rtes := rte_eg.exceptions) == 1
-        assert 'asyncio-side' in rtes[0].args[0]
+        patt: str = 'asyncio-side'
+        expect_exc = RuntimeError
+
+    with pytest.raises(expect_exc) as excinfo:
+        tractor.to_asyncio.run_as_asyncio_guest(
+            trio_main=_trio_main,
+        )
+
+    caught_exc = excinfo.value
+    assert patt in caught_exc.args

tests/test_root_runtime.py

@@ -83,3 +83,26 @@ def test_implicit_root_via_first_nursery(
             assert tractor.current_actor().aid.name == 'root'
 
     trio.run(main)
+
+
+def test_runtime_vars_unset(
+    reg_addr: tuple,
+    debug_mode: bool
+):
+    '''
+    Ensure any `._state._runtime_vars` are restored to default values
+    after the root actor-runtime exits!
+
+    '''
+    assert not tractor._state._runtime_vars['_debug_mode']
+    async def main():
+        assert not tractor._state._runtime_vars['_debug_mode']
+        async with tractor.open_nursery(
+            debug_mode=True,
+        ):
+            assert tractor._state._runtime_vars['_debug_mode']
+
+        # after runtime closure, should be reverted!
+        assert not tractor._state._runtime_vars['_debug_mode']
+
+    trio.run(main)

tests/test_trioisms.py

@@ -6,10 +6,18 @@ want to see changed.
 from contextlib import (
     asynccontextmanager as acm,
 )
+from types import ModuleType
+
+from functools import partial
 
 import pytest
+from _pytest import pathlib
+from tractor.trionics import collapse_eg
 import trio
 from trio import TaskStatus
+from tractor._testing import (
+    examples_dir,
+)
 
 
 @pytest.mark.parametrize(
@@ -64,9 +72,8 @@ def test_stashed_child_nursery(use_start_soon):
     async def main():
 
         async with (
-            trio.open_nursery(
-                strict_exception_groups=False,
-            ) as pn,
+            collapse_eg(),
+            trio.open_nursery() as pn,
         ):
             cn = await pn.start(mk_child_nursery)
             assert cn
@@ -106,66 +113,21 @@ def test_acm_embedded_nursery_propagates_enter_err(
     debug_mode: bool,
 ):
     '''
-    Demo how a masking `trio.Cancelled` could be handled by unmasking from the
-    `.__context__` field when a user (by accident) re-raises from a `finally:`.
+    Demo how a masking `trio.Cancelled` could be handled by unmasking
+    from the `.__context__` field when a user (by accident) re-raises
+    from a `finally:`.
 
     '''
     import tractor
 
-    @acm
-    async def maybe_raise_from_masking_exc(
-        tn: trio.Nursery,
-        unmask_from: BaseException|None = trio.Cancelled
-
-        # TODO, maybe offer a collection?
-        # unmask_from: set[BaseException] = {
-        #     trio.Cancelled,
-        # },
-    ):
-        if not unmask_from:
-            yield
-            return
-
-        try:
-            yield
-        except* unmask_from as be_eg:
-
-            # TODO, if we offer `unmask_from: set`
-            # for masker_exc_type in unmask_from:
-
-            matches, rest = be_eg.split(unmask_from)
-            if not matches:
-                raise
-
-            for exc_match in be_eg.exceptions:
-                if (
-                    (exc_ctx := exc_match.__context__)
-                    and
-                    type(exc_ctx) not in {
-                        # trio.Cancelled,  # always by default?
-                        unmask_from,
-                    }
-                ):
-                    exc_ctx.add_note(
-                        f'\n'
-                        f'WARNING: the above error was masked by a {unmask_from!r} !?!\n'
-                        f'Are you always cancelling? Say from a `finally:` ?\n\n'
-
-                        f'{tn!r}'
-                    )
-                    raise exc_ctx from exc_match
-
-
     @acm
     async def wraps_tn_that_always_cancels():
         async with (
             trio.open_nursery() as tn,
-            maybe_raise_from_masking_exc(
-                tn=tn,
+            tractor.trionics.maybe_raise_from_masking_exc(
                 unmask_from=(
-                    trio.Cancelled
-                    if unmask_from_canc
-                    else None
+                    (trio.Cancelled,) if unmask_from_canc
+                    else ()
                 ),
             )
         ):
@@ -180,8 +142,7 @@ def test_acm_embedded_nursery_propagates_enter_err(
         with tractor.devx.maybe_open_crash_handler(
             pdb=debug_mode,
         ) as bxerr:
-            if bxerr:
-                assert not bxerr.value
+            assert not bxerr.value
 
             async with (
                 wraps_tn_that_always_cancels() as tn,
@@ -189,11 +150,12 @@ def test_acm_embedded_nursery_propagates_enter_err(
                 assert not tn.cancel_scope.cancel_called
                 assert 0
 
-        assert (
-            (err := bxerr.value)
-            and
-            type(err) is AssertionError
-        )
+        if debug_mode:
+            assert (
+                (err := bxerr.value)
+                and
+                type(err) is AssertionError
+            )
 
     with pytest.raises(ExceptionGroup) as excinfo:
         trio.run(_main)
@@ -202,3 +164,139 @@ def test_acm_embedded_nursery_propagates_enter_err(
     assert_eg, rest_eg = eg.split(AssertionError)
 
     assert len(assert_eg.exceptions) == 1
+
+
+def test_gatherctxs_with_memchan_breaks_multicancelled(
+    debug_mode: bool,
+):
+    '''
+    Demo how a using an `async with sndchan` inside
+    a `.trionics.gather_contexts()` task will break a strict-eg-tn's
+    multi-cancelled absorption..
+
+    '''
+    from tractor import (
+        trionics,
+    )
+
+    @acm
+    async def open_memchan() -> trio.abc.ReceiveChannel:
+
+        task: trio.Task = trio.lowlevel.current_task()
+        print(
+            f'Opening {task!r}\n'
+        )
+
+        # 1 to force eager sending
+        send, recv = trio.open_memory_channel(16)
+
+        try:
+            async with send:
+                yield recv
+        finally:
+            print(
+                f'Closed {task!r}\n'
+            )
+
+    async def main():
+        async with (
+            # XXX should ensure ONLY the KBI
+            # is relayed upward
+            collapse_eg(),
+            trio.open_nursery(), # as tn,
+
+            trionics.gather_contexts([
+                open_memchan(),
+                open_memchan(),
+            ]) as recv_chans,
+        ):
+            assert len(recv_chans) == 2
+
+            await trio.sleep(1)
+            raise KeyboardInterrupt
+            # tn.cancel_scope.cancel()
+
+    with pytest.raises(KeyboardInterrupt):
+        trio.run(main)
+
+
+@pytest.mark.parametrize(
+    'raise_unmasked', [
+        True,
+        pytest.param(
+            False,
+            marks=pytest.mark.xfail(
+                reason="see examples/trio/send_chan_aclose_masks.py"
+            )
+        ),
+    ]
+)
+@pytest.mark.parametrize(
+    'child_errors_mid_stream',
+    [True, False],
+)
+def test_unmask_aclose_as_checkpoint_on_aexit(
+    raise_unmasked: bool,
+    child_errors_mid_stream: bool,
+    debug_mode: bool,
+):
+    '''
+    Verify that our unmasker util works over the common case where
+    a mem-chan's `.aclose()` is included in an `@acm` stack
+    and it being currently a checkpoint, can `trio.Cancelled`-mask an embedded
+    exception from user code resulting in a silent failure which
+    appears like graceful cancellation.
+
+    This test suite is mostly implemented as an example script so it
+    could more easily be shared with `trio`-core peeps as `tractor`-less
+    minimum reproducing example.
+
+    '''
+    mod: ModuleType = pathlib.import_path(
+        examples_dir()
+        / 'trio'
+        / 'send_chan_aclose_masks_beg.py',
+        root=examples_dir(),
+        consider_namespace_packages=False,
+    )
+    with pytest.raises(RuntimeError):
+        trio.run(partial(
+            mod.main,
+            raise_unmasked=raise_unmasked,
+            child_errors_mid_stream=child_errors_mid_stream,
+        ))
+
+
+
+@pytest.mark.parametrize(
+    'ignore_special_cases', [
+        True,
+        pytest.param(
+            False,
+            marks=pytest.mark.xfail(
+                reason="see examples/trio/lockacquire_not_umasked.py"
+            )
+        ),
+    ]
+)
+def test_cancelled_lockacquire_in_ipctx_not_unmasked(
+    ignore_special_cases: bool,
+    loglevel: str,
+    debug_mode: bool,
+):
+    mod: ModuleType = pathlib.import_path(
+        examples_dir()
+        / 'trio'
+        / 'lockacquire_not_unmasked.py',
+        root=examples_dir(),
+        consider_namespace_packages=False,
+    )
+    async def _main():
+        with trio.fail_after(2):
+            await mod.main(
+                ignore_special_cases=ignore_special_cases,
+                loglevel=loglevel,
+                debug_mode=debug_mode,
+            )
+
+    trio.run(_main)

tractor/_clustering.py

@@ -55,10 +55,17 @@ async def open_actor_cluster(
         raise ValueError(
             'Number of names is {len(names)} but count it {count}')
 
-    async with tractor.open_nursery(
-        **runtime_kwargs,
-    ) as an:
-        async with trio.open_nursery() as n:
+    async with (
+        # tractor.trionics.collapse_eg(),
+        tractor.open_nursery(
+            **runtime_kwargs,
+        ) as an
+    ):
+        async with (
+            # tractor.trionics.collapse_eg(),
+            trio.open_nursery() as tn,
+            tractor.trionics.maybe_raise_from_masking_exc()
+        ):
             uid = tractor.current_actor().uid
 
             async def _start(name: str) -> None:
@@ -69,9 +76,8 @@ async def open_actor_cluster(
                 )
 
             for name in names:
-                n.start_soon(_start, name)
+                tn.start_soon(_start, name)
 
         assert len(portals) == count
         yield portals
-
         await an.cancel(hard_kill=hard_kill)

tractor/_context.py

@@ -101,6 +101,9 @@ from ._state import (
     debug_mode,
     _ctxvar_Context,
 )
+from .trionics import (
+    collapse_eg,
+)
 # ------ - ------
 if TYPE_CHECKING:
     from ._portal import Portal
@@ -151,7 +154,7 @@ class Context:
     2 cancel-scope-linked, communicating and parallel executing
     `Task`s. Contexts are allocated on each side of any task
     RPC-linked msg dialog, i.e. for every request to a remote
-    actor from a `Portal`. On the "callee" side a context is
+    actor from a `Portal`. On the "child" side a context is
     always allocated inside `._rpc._invoke()`.
 
     TODO: more detailed writeup on cancellation, error and
@@ -219,8 +222,8 @@ class Context:
     # `._runtime.invoke()`.
     _remote_func_type: str | None = None
 
-    # NOTE: (for now) only set (a portal) on the caller side since
-    # the callee doesn't generally need a ref to one and should
+    # NOTE: (for now) only set (a portal) on the parent side since
+    # the child doesn't generally need a ref to one and should
     # normally need to explicitly ask for handle to its peer if
     # more the the `Context` is needed?
     _portal: Portal | None = None
@@ -249,12 +252,12 @@ class Context:
     _outcome_msg: Return|Error|ContextCancelled = Unresolved
 
     # on a clean exit there should be a final value
-    # delivered from the far end "callee" task, so
+    # delivered from the far end "child" task, so
     # this value is only set on one side.
     # _result: Any | int = None
     _result: PayloadT|Unresolved = Unresolved
 
-    # if the local "caller"  task errors this value is always set
+    # if the local "parent"  task errors this value is always set
     # to the error that was captured in the
     # `Portal.open_context().__aexit__()` teardown block OR, in
     # 2 special cases when an (maybe) expected remote error
@@ -290,9 +293,9 @@ class Context:
     # a `ContextCancelled` due to a call to `.cancel()` triggering
     # "graceful closure" on either side:
     # - `._runtime._invoke()` will check this flag before engaging
-    #   the crash handler REPL in such cases where the "callee"
+    #   the crash handler REPL in such cases where the "child"
     #   raises the cancellation,
-    # - `.devx._debug.lock_stdio_for_peer()` will set it to `False` if
+    # - `.devx.debug.lock_stdio_for_peer()` will set it to `False` if
     #   the global tty-lock has been configured to filter out some
     #   actors from being able to acquire the debugger lock.
     _enter_debugger_on_cancel: bool = True
@@ -304,8 +307,8 @@ class Context:
     _stream_opened: bool = False
     _stream: MsgStream|None = None
 
-    # caller of `Portal.open_context()` for
-    # logging purposes mostly
+    # the parent-task's calling-fn's frame-info, the frame above
+    # `Portal.open_context()`, for introspection/logging.
     _caller_info: CallerInfo|None = None
 
     # overrun handling machinery
@@ -439,25 +442,25 @@ class Context:
         '''
         Records whether cancellation has been requested for this context
         by a call to  `.cancel()` either due to,
-        - either an explicit call by some local task,
+        - an explicit call by some local task,
         - or an implicit call due to an error caught inside
-          the ``Portal.open_context()`` block.
+          the `Portal.open_context()` block.
 
         '''
         return self._cancel_called
 
-    @cancel_called.setter
-    def cancel_called(self, val: bool) -> None:
-        '''
-        Set the self-cancelled request `bool` value.
+    # XXX, to debug who frickin sets it..
+    # @cancel_called.setter
+    # def cancel_called(self, val: bool) -> None:
+    #     '''
+    #     Set the self-cancelled request `bool` value.
 
-        '''
-        # to debug who frickin sets it..
-        # if val:
-        #     from .devx import pause_from_sync
-        #     pause_from_sync()
+    #     '''
+    #     if val:
+    #         from .devx import pause_from_sync
+    #         pause_from_sync()
 
-        self._cancel_called = val
+    #     self._cancel_called = val
 
     @property
     def canceller(self) -> tuple[str, str]|None:
@@ -526,11 +529,11 @@ class Context:
         '''
         Exactly the value of `self._scope.cancelled_caught`
         (delegation) and should only be (able to be read as)
-        `True` for a `.side == "caller"` ctx wherein the
+        `True` for a `.side == "parent"` ctx wherein the
         `Portal.open_context()` block was exited due to a call to
         `._scope.cancel()` - which should only ocurr in 2 cases:
 
-        - a caller side calls `.cancel()`, the far side cancels
+        - a parent side calls `.cancel()`, the far side cancels
           and delivers back a `ContextCancelled` (making
           `.cancel_acked == True`) and `._scope.cancel()` is
           called by `._maybe_cancel_and_set_remote_error()` which
@@ -539,20 +542,20 @@ class Context:
           => `._scope.cancelled_caught == True` by normal `trio`
           cs semantics.
 
-        - a caller side is delivered a `._remote_error:
+        - a parent side is delivered a `._remote_error:
           RemoteActorError` via `._deliver_msg()` and a transitive
           call to `_maybe_cancel_and_set_remote_error()` calls
           `._scope.cancel()` and that cancellation eventually
           results in `trio.Cancelled`(s) caught in the
           `.open_context()` handling around the @acm's `yield`.
 
-        Only as an FYI, in the "callee" side case it can also be
+        Only as an FYI, in the "child" side case it can also be
         set but never is readable by any task outside the RPC
         machinery in `._invoke()` since,:
-        - when a callee side calls `.cancel()`, `._scope.cancel()`
+        - when a child side calls `.cancel()`, `._scope.cancel()`
           is called immediately and handled specially inside
           `._invoke()` to raise a `ContextCancelled` which is then
-          sent to the caller side.
+          sent to the parent side.
 
           However, `._scope.cancelled_caught` can NEVER be
           accessed/read as `True` by any RPC invoked task since it
@@ -632,6 +635,71 @@ class Context:
         '''
         await self.chan.send(Stop(cid=self.cid))
 
+    @property
+    def parent_task(self) -> trio.Task:
+        '''
+        This IPC context's "owning task" which is a `trio.Task`
+        on one of the "sides" of the IPC.
+
+        Note that the "parent_" prefix here refers to the local
+        `trio` task tree using the same interface as
+        `trio.Nursery.parent_task` whereas for IPC contexts,
+        a different cross-actor task hierarchy exists:
+
+        - a "parent"-side which originally entered
+          `Portal.open_context()`,
+
+        - the "child"-side which was spawned and scheduled to invoke
+          a function decorated with `@tractor.context`.
+
+        This task is thus a handle to mem-domain-distinct/per-process
+        `Nursery.parent_task` depending on in which of the above
+        "sides" this context exists.
+
+        '''
+        return self._task
+
+    def _is_blocked_on_rx_chan(self) -> bool:
+        '''
+        Predicate to indicate whether the owner `._task: trio.Task` is
+        currently blocked (by `.receive()`-ing) on its underlying RPC
+        feeder `._rx_chan`.
+
+        This knowledge is highly useful when handling so called
+        "out-of-band" (OoB) cancellation conditions where a peer
+        actor's task transmitted some remote error/cancel-msg and we
+        must know whether to signal-via-cancel currently executing
+        "user-code" (user defined code embedded in `ctx._scope`) or
+        simply to forward the IPC-msg-as-error **without calling**
+        `._scope.cancel()`.
+
+        In the latter case it is presumed that if the owner task is
+        blocking for the next IPC msg, it will eventually receive,
+        process and raise the equivalent local error **without**
+        requiring `._scope.cancel()` to be explicitly called by the
+        *delivering OoB RPC-task* (via `_deliver_msg()`).
+
+        '''
+        # NOTE, see the mem-chan meth-impls for *why* this
+        # logic works,
+        # `trio._channel.MemoryReceiveChannel.receive[_nowait]()`
+        #
+        # XXX realize that this is NOT an
+        # official/will-be-loudly-deprecated API:
+        # - https://trio.readthedocs.io/en/stable/reference-lowlevel.html#trio.lowlevel.Task.custom_sleep_data
+        #  |_https://trio.readthedocs.io/en/stable/reference-lowlevel.html#trio.lowlevel.wait_task_rescheduled
+        #
+        # orig repo intro in the mem-chan change over patch:
+        # - https://github.com/python-trio/trio/pull/586#issuecomment-414039117
+        #  |_https://github.com/python-trio/trio/pull/616
+        #  |_https://github.com/njsmith/trio/commit/98c38cef6f62e731bf8c7190e8756976bface8f0
+        #
+        return (
+            self._task.custom_sleep_data
+            is
+            self._rx_chan
+        )
+
     def _maybe_cancel_and_set_remote_error(
         self,
         error: BaseException,
@@ -663,7 +731,7 @@ class Context:
         when called/closed by actor local task(s).
 
         NOTEs: 
-          - It is expected that the caller has previously unwrapped
+          - It is expected that the parent has previously unwrapped
             the remote error using a call to `unpack_error()` and
             provides that output exception value as the input
             `error` argument *here*.
@@ -673,7 +741,7 @@ class Context:
             `Portal.open_context()` (ideally) we want to interrupt
             any ongoing local tasks operating within that
             `Context`'s cancel-scope so as to be notified ASAP of
-            the remote error and engage any caller handling (eg.
+            the remote error and engage any parent handling (eg.
             for cross-process task supervision).
 
           - In some cases we may want to raise the remote error
@@ -740,6 +808,8 @@ class Context:
             # cancelled, NOT their reported canceller. IOW in the
             # latter case we're cancelled by someone else getting
             # cancelled.
+            #
+            # !TODO, switching to `Actor.aid` here!
             if (canc := error.canceller) == self._actor.uid:
                 whom: str = 'us'
                 self._canceller = canc
@@ -782,13 +852,27 @@ class Context:
         if self._canceller is None:
             log.error('Ctx has no canceller set!?')
 
+        cs: trio.CancelScope = self._scope
+
+        # ?TODO? see comment @ .start_remote_task()`
+        #
+        # if not cs:
+        #     from .devx import mk_pdb
+        #     mk_pdb().set_trace()
+        #     raise RuntimeError(
+        #         f'IPC ctx was not be opened prior to remote error delivery !?\n'
+        #         f'{self}\n'
+        #         f'\n'
+        #         f'`Portal.open_context()` must be entered (somewhere) beforehand!\n'
+        #     )
+
         # Cancel the local `._scope`, catch that
         # `._scope.cancelled_caught` and re-raise any remote error
         # once exiting (or manually calling `.wait_for_result()`) the
         # `.open_context()`  block.
-        cs: trio.CancelScope = self._scope
         if (
             cs
+            and not cs.cancel_called
 
             # XXX this is an expected cancel request response
             # message and we **don't need to raise it** in the
@@ -797,8 +881,7 @@ class Context:
             # if `._cancel_called` then `.cancel_acked and .cancel_called`
             # always should be set.
             and not self._is_self_cancelled()
-            and not cs.cancel_called
-            and not cs.cancelled_caught
+            # and not cs.cancelled_caught
         ):
             if (
                 msgerr
@@ -809,7 +892,7 @@ class Context:
                 not self._cancel_on_msgerr
             ):
                 message: str = (
-                    'NOT Cancelling `Context._scope` since,\n'
+                    f'NOT Cancelling `Context._scope` since,\n'
                     f'Context._cancel_on_msgerr = {self._cancel_on_msgerr}\n\n'
                     f'AND we got a msg-type-error!\n'
                     f'{error}\n'
@@ -819,13 +902,43 @@ class Context:
                 # `trio.Cancelled` subtype here ;)
                 # https://github.com/goodboy/tractor/issues/368
                 message: str = 'Cancelling `Context._scope` !\n\n'
-                # from .devx import pause_from_sync
-                # pause_from_sync()
-                self._scope.cancel()
-        else:
-            message: str = 'NOT cancelling `Context._scope` !\n\n'
+                cs.cancel()
+
+        # TODO, explicit condition for OoB (self-)cancellation?
+        # - we called `Portal.cancel_actor()` from this actor
+        #   and the peer ctx task delivered ctxc due to it.
+        # - currently `self._is_self_cancelled()` will be true
+        #   since the ctxc.canceller check will match us even though it
+        #   wasn't from this ctx specifically!
+        elif (
+            cs
+            and self._is_self_cancelled()
+            and not cs.cancel_called
+        ):
+            message: str = (
+                'Cancelling `ctx._scope` due to OoB self-cancel ?!\n'
+                '\n'
+            )
             # from .devx import mk_pdb
             # mk_pdb().set_trace()
+            # TODO XXX, required to fix timeout failure in
+            # `test_cancelled_lockacquire_in_ipctx_not_unmaskeed`
+            #
+
+            # XXX NOTE XXX, this is SUPER SUBTLE!
+            # we only want to cancel our embedded `._scope`
+            # if the ctx's current/using task is NOT blocked
+            # on `._rx_chan.receive()` and on some other
+            # `trio`-checkpoint since in the former case
+            # any `._remote_error` will be relayed through
+            # the rx-chan and appropriately raised by the owning
+            # `._task` directly. IF the owner task is however
+            # blocking elsewhere we need to interrupt it **now**.
+            if not self._is_blocked_on_rx_chan():
+                cs.cancel()
+        else:
+            # rx_stats = self._rx_chan.statistics()
+            message: str = 'NOT cancelling `Context._scope` !\n\n'
 
         fmt_str: str = 'No `self._scope: CancelScope` was set/used ?\n'
         if (
@@ -849,6 +962,7 @@ class Context:
                 +
                 cs_fmt
             )
+
         log.cancel(
             message
             +
@@ -881,6 +995,11 @@ class Context:
 
     @property
     def repr_caller(self) -> str:
+        '''
+        Render a "namespace-path" style representation of the calling
+        task-fn.
+
+        '''
         ci: CallerInfo|None = self._caller_info
         if ci:
             return (
@@ -894,7 +1013,7 @@ class Context:
     def repr_api(self) -> str:
         return 'Portal.open_context()'
 
-        # TODO: use `.dev._frame_stack` scanning to find caller!
+        # TODO: use `.dev._frame_stack` scanning to find caller fn!
         # ci: CallerInfo|None = self._caller_info
         # if ci:
         #     return (
@@ -929,18 +1048,19 @@ class Context:
         => That is, an IPC `Context` (this) **does not**
            have the same semantics as a `trio.CancelScope`.
 
-        If the caller (who entered the `Portal.open_context()`)
+        If the parent (who entered the `Portal.open_context()`)
         desires that the internal block's cancel-scope  be
         cancelled it should open its own `trio.CancelScope` and
         manage it as needed.
 
         '''
         side: str = self.side
-        # XXX for debug via the `@.setter`
-        self.cancel_called = True
+        self._cancel_called = True
+        # ^ XXX for debug via the `@.setter`
+        # self.cancel_called = True
 
         header: str = (
-            f'Cancelling ctx from {side.upper()}-side\n'
+            f'Cancelling ctx from {side!r}-side\n'
         )
         reminfo: str = (
             # ' =>\n'
@@ -948,7 +1068,7 @@ class Context:
             f'\n'
             f'c)=> {self.chan.uid}\n'
             f'   |_[{self.dst_maddr}\n'
-            f'     >>{self.repr_rpc}\n'
+            f'     >> {self.repr_rpc}\n'
             # f'    >> {self._nsf}() -> {codec}[dict]:\n\n'
             # TODO: pull msg-type from spec re #320
         )
@@ -1001,7 +1121,6 @@ class Context:
                 else:
                     log.cancel(
                         f'Timed out on cancel request of remote task?\n'
-                        f'\n'
                         f'{reminfo}'
                     )
 
@@ -1012,7 +1131,7 @@ class Context:
         # `_invoke()` RPC task.
         #
         # NOTE: on this side we ALWAYS cancel the local scope
-        # since the caller expects a `ContextCancelled` to be sent
+        # since the parent expects a `ContextCancelled` to be sent
         # from `._runtime._invoke()` back to the other side. The
         # logic for catching the result of the below
         # `._scope.cancel()` is inside the `._runtime._invoke()`
@@ -1185,8 +1304,8 @@ class Context:
 
     ) -> Any|Exception:
         '''
-        From some (caller) side task, wait for and return the final
-        result from the remote (callee) side's task.
+        From some (parent) side task, wait for and return the final
+        result from the remote (child) side's task.
 
         This provides a mechanism for one task running in some actor to wait
         on another task at the other side, in some other actor, to terminate.
@@ -1251,8 +1370,8 @@ class Context:
 
             # ?XXX, should already be set in `._deliver_msg()` right?
             if self._outcome_msg is not Unresolved:
-                # from .devx import _debug
-                # await _debug.pause()
+                # from .devx import debug
+                # await debug.pause()
                 assert self._outcome_msg is outcome_msg
             else:
                 self._outcome_msg = outcome_msg
@@ -1482,6 +1601,12 @@ class Context:
                 ):
                     status = 'peer-cancelled'
 
+            case (
+                Unresolved,
+                trio.Cancelled(),  # any error-type
+            ) if self.canceller:
+                status = 'actor-cancelled'
+
             # (remote) error condition
             case (
                 Unresolved,
@@ -1595,7 +1720,7 @@ class Context:
                 raise err
 
         # TODO: maybe a flag to by-pass encode op if already done
-        # here in caller?
+        # here in parent?
         await self.chan.send(started_msg)
 
         # set msg-related internal runtime-state
@@ -1671,7 +1796,7 @@ class Context:
 
          XXX RULES XXX
         ------ - ------
-        - NEVER raise remote errors from this method; a runtime task caller.
+        - NEVER raise remote errors from this method; a calling runtime-task.
           An error "delivered" to a ctx should always be raised by
           the corresponding local task operating on the
           `Portal`/`Context` APIs.
@@ -1747,7 +1872,7 @@ class Context:
 
             else:
                 report = (
-                    'Queueing OVERRUN msg on caller task:\n\n'
+                    'Queueing OVERRUN msg on parent task:\n\n'
                     + report
                 )
                 log.debug(report)
@@ -1943,12 +2068,12 @@ async def open_context_from_portal(
     IPC protocol.
 
     The yielded `tuple` is a pair delivering a `tractor.Context`
-    and any first value "sent" by the "callee" task via a call
+    and any first value "sent" by the "child" task via a call
     to `Context.started(<value: Any>)`; this side of the
-    context does not unblock until the "callee" task calls
+    context does not unblock until the "child" task calls
     `.started()` in similar style to `trio.Nursery.start()`.
-    When the "callee" (side that is "called"/started by a call
-    to *this* method) returns, the caller side (this) unblocks
+    When the "child" (side that is "called"/started by a call
+    to *this* method) returns, the parent side (this) unblocks
     and any final value delivered from the other end can be
     retrieved using the `Contex.wait_for_result()` api.
 
@@ -1961,7 +2086,7 @@ async def open_context_from_portal(
     __tracebackhide__: bool = hide_tb
 
     # denote this frame as a "runtime frame" for stack
-    # introspection where we report the caller code in logging
+    # introspection where we report the parent code in logging
     # and error message content.
     # NOTE: 2 bc of the wrapping `@acm`
     __runtimeframe__: int = 2  # noqa
@@ -1996,6 +2121,9 @@ async def open_context_from_portal(
             f'|_{portal.actor}\n'
         )
 
+    # ?TODO? could we move this to inside the `tn` block?
+    # -> would allow doing `ctx.parent_task = tn.parent_task` ?
+    # -> would allow a `if not ._scope: => raise RTE` ?
     ctx: Context = await portal.actor.start_remote_task(
         portal.channel,
         nsf=nsf,
@@ -2020,13 +2148,12 @@ async def open_context_from_portal(
     # placeholder for any exception raised in the runtime
     # or by user tasks which cause this context's closure.
     scope_err: BaseException|None = None
-    ctxc_from_callee: ContextCancelled|None = None
+    ctxc_from_child: ContextCancelled|None = None
     try:
+        # from .devx import pause
         async with (
-            trio.open_nursery(
-                strict_exception_groups=False,
-            ) as tn,
-
+            collapse_eg(),
+            trio.open_nursery() as tn,
             msgops.maybe_limit_plds(
                 ctx=ctx,
                 spec=ctx_meta.get('pld_spec'),
@@ -2046,6 +2173,10 @@ async def open_context_from_portal(
             # the dialog, the `Error` msg should be raised from the `msg`
             # handling block below.
             try:
+                log.runtime(
+                    f'IPC ctx parent waiting on Started msg..\n'
+                    f'ctx.cid: {ctx.cid!r}\n'
+                )
                 started_msg, first = await ctx._pld_rx.recv_msg(
                     ipc=ctx,
                     expect_msg=Started,
@@ -2054,16 +2185,16 @@ async def open_context_from_portal(
                 )
             except trio.Cancelled as taskc:
                 ctx_cs: trio.CancelScope = ctx._scope
+                log.cancel(
+                    f'IPC ctx was cancelled during "child" task sync due to\n\n'
+                    f'.cid: {ctx.cid!r}\n'
+                    f'.maybe_error: {ctx.maybe_error!r}\n'
+                )
+                # await pause(shield=True)
+
                 if not ctx_cs.cancel_called:
                     raise
 
-                # from .devx import pause
-                # await pause(shield=True)
-
-                log.cancel(
-                    'IPC ctx was cancelled during "child" task sync due to\n\n'
-                    f'{ctx.maybe_error}\n'
-                )
                 # OW if the ctx's scope was cancelled manually,
                 # likely the `Context` was cancelled via a call to
                 # `._maybe_cancel_and_set_remote_error()` so ensure
@@ -2101,7 +2232,7 @@ async def open_context_from_portal(
             # that we can re-use it around the `yield` ^ here
             # or vice versa?
             #
-            # maybe TODO NOTE: between the caller exiting and
+            # maybe TODO NOTE: between the parent exiting and
             # arriving here the far end may have sent a ctxc-msg or
             # other error, so the quetion is whether we should check
             # for it here immediately and maybe raise so as to engage
@@ -2167,16 +2298,16 @@ async def open_context_from_portal(
     #   request in which case we DO let the error bubble to the
     #   opener.
     #
-    # 2-THIS "caller" task somewhere invoked `Context.cancel()`
-    #   and received a `ContextCanclled` from the "callee"
+    # 2-THIS "parent" task somewhere invoked `Context.cancel()`
+    #   and received a `ContextCanclled` from the "child"
     #   task, in which case we mask the `ContextCancelled` from
-    #   bubbling to this "caller" (much like how `trio.Nursery`
+    #   bubbling to this "parent" (much like how `trio.Nursery`
     #   swallows any `trio.Cancelled` bubbled by a call to
     #   `Nursery.cancel_scope.cancel()`)
     except ContextCancelled as ctxc:
         scope_err = ctxc
         ctx._local_error: BaseException = scope_err
-        ctxc_from_callee = ctxc
+        ctxc_from_child = ctxc
 
         # XXX TODO XXX: FIX THIS debug_mode BUGGGG!!!
         # using this code and then resuming the REPL will
@@ -2186,8 +2317,8 @@ async def open_context_from_portal(
         #   documenting it as a definittive example of
         #   debugging the tractor-runtime itself using it's
         #   own `.devx.` tooling!
-        # 
-        # await _debug.pause()
+        #
+        # await debug.pause()
 
         # CASE 2: context was cancelled by local task calling
         # `.cancel()`, we don't raise and the exit block should
@@ -2213,11 +2344,11 @@ async def open_context_from_portal(
     # the above `._scope` can be cancelled due to:
     # 1. an explicit self cancel via `Context.cancel()` or
     #    `Actor.cancel()`,
-    # 2. any "callee"-side remote error, possibly also a cancellation
+    # 2. any "child"-side remote error, possibly also a cancellation
     #    request by some peer,
-    # 3. any "caller" (aka THIS scope's) local error raised in the above `yield`
+    # 3. any "parent" (aka THIS scope's) local error raised in the above `yield`
     except (
-        # CASE 3: standard local error in this caller/yieldee
+        # CASE 3: standard local error in this parent/yieldee
         Exception,
 
         # CASES 1 & 2: can manifest as a `ctx._scope_nursery`
@@ -2231,9 +2362,9 @@ async def open_context_from_portal(
         #   any `Context._maybe_raise_remote_err()` call.
         #
         # 2.-`BaseExceptionGroup[ContextCancelled | RemoteActorError]`
-        #    from any error delivered from the "callee" side
+        #    from any error delivered from the "child" side
         #    AND a group-exc is only raised if there was > 1
-        #    tasks started *here* in the "caller" / opener
+        #    tasks started *here* in the "parent" / opener
         #    block. If any one of those tasks calls
         #    `.wait_for_result()` or `MsgStream.receive()`
         #    `._maybe_raise_remote_err()` will be transitively
@@ -2246,34 +2377,37 @@ async def open_context_from_portal(
         trio.Cancelled,  # NOTE: NOT from inside the ctx._scope
         KeyboardInterrupt,
 
-    ) as caller_err:
-        scope_err = caller_err
+    ) as rent_err:
+        scope_err = rent_err
         ctx._local_error: BaseException = scope_err
 
         # XXX: ALWAYS request the context to CANCEL ON any ERROR.
         # NOTE: `Context.cancel()` is conversely NEVER CALLED in
         # the `ContextCancelled` "self cancellation absorbed" case
         # handled in the block above ^^^ !!
-        # await _debug.pause()
+        # await debug.pause()
         # log.cancel(
         match scope_err:
-            case trio.Cancelled:
+            case trio.Cancelled():
                 logmeth = log.cancel
+                cause: str = 'cancelled'
 
             # XXX explicitly report on any non-graceful-taskc cases
             case _:
+                cause: str = 'errored'
                 logmeth = log.exception
 
         logmeth(
-            f'ctx {ctx.side!r}-side exited with {ctx.repr_outcome()}\n'
+            f'ctx {ctx.side!r}-side {cause!r} with,\n'
+            f'{ctx.repr_outcome()!r}\n'
         )
 
         if debug_mode():
-            # async with _debug.acquire_debug_lock(portal.actor.uid):
+            # async with debug.acquire_debug_lock(portal.actor.uid):
             #     pass
             # TODO: factor ^ into below for non-root cases?
             #
-            from .devx._debug import maybe_wait_for_debugger
+            from .devx.debug import maybe_wait_for_debugger
             was_acquired: bool = await maybe_wait_for_debugger(
                 # header_msg=(
                 #     'Delaying `ctx.cancel()` until debug lock '
@@ -2286,10 +2420,11 @@ async def open_context_from_portal(
                     'Calling `ctx.cancel()`!\n'
                 )
 
-        # we don't need to cancel the callee if it already
+        # we don't need to cancel the child if it already
         # told us it's cancelled ;p
-        if ctxc_from_callee is None:
+        if ctxc_from_child is None:
             try:
+                # await pause(shield=True)
                 await ctx.cancel()
             except (
                 trio.BrokenResourceError,
@@ -2319,8 +2454,8 @@ async def open_context_from_portal(
             # via a call to
             # `Context._maybe_cancel_and_set_remote_error()`.
             # As per `Context._deliver_msg()`, that error IS
-            # ALWAYS SET any time "callee" side fails and causes "caller
-            # side" cancellation via a `ContextCancelled` here.
+            # ALWAYS SET any time "child" side fails and causes
+            # "parent side" cancellation via a `ContextCancelled` here.
             try:
                 result_or_err: Exception|Any = await ctx.wait_for_result()
             except BaseException as berr:
@@ -2336,8 +2471,8 @@ async def open_context_from_portal(
                 raise
 
             # yes this worx!
-            # from .devx import _debug
-            # await _debug.pause()
+            # from .devx import debug
+            # await debug.pause()
 
             # an exception type boxed in a `RemoteActorError`
             # is returned (meaning it was obvi not raised)
@@ -2356,7 +2491,7 @@ async def open_context_from_portal(
                     )
                 case (None, _):
                     log.runtime(
-                        'Context returned final result from callee task:\n'
+                        'Context returned final result from child task:\n'
                         f'<= peer: {uid}\n'
                         f'  |_ {nsf}()\n\n'
 
@@ -2372,7 +2507,7 @@ async def open_context_from_portal(
         # where the root is waiting on the lock to clear but the
         # child has already cleared it and clobbered IPC.
         if debug_mode():
-            from .devx._debug import maybe_wait_for_debugger
+            from .devx.debug import maybe_wait_for_debugger
             await maybe_wait_for_debugger()
 
         # though it should be impossible for any tasks
@@ -2446,12 +2581,14 @@ async def open_context_from_portal(
                 log.cancel(
                     f'Context cancelled by local {ctx.side!r}-side task\n'
                     f'c)>\n'
-                    f' |_{ctx._task}\n\n'
-                    f'{repr(scope_err)}\n'
+                    f'  |_{ctx.parent_task}\n'
+                    f'   .cid={ctx.cid!r}\n'
+                    f'\n'
+                    f'{scope_err!r}\n'
                 )
 
             # TODO: should we add a `._cancel_req_received`
-            # flag to determine if the callee manually called
+            # flag to determine if the child manually called
             # `ctx.cancel()`?
             # -[ ] going to need a cid check no?
 
@@ -2507,7 +2644,7 @@ def mk_context(
     recv_chan: trio.MemoryReceiveChannel
     send_chan, recv_chan = trio.open_memory_channel(msg_buffer_size)
 
-    # TODO: only scan caller-info if log level so high!
+    # TODO: only scan parent-info if log level so high!
     from .devx._frame_stack import find_caller_info
     caller_info: CallerInfo|None = find_caller_info()
 

tractor/_discovery.py

@@ -28,7 +28,10 @@ from typing import (
 from contextlib import asynccontextmanager as acm
 
 from tractor.log import get_logger
-from .trionics import gather_contexts
+from .trionics import (
+    gather_contexts,
+    collapse_eg,
+)
 from .ipc import _connect_chan, Channel
 from ._addr import (
     UnwrappedAddress,
@@ -87,7 +90,6 @@ async def get_registry(
             yield regstr_ptl
 
 
-
 @acm
 async def get_root(
     **kwargs,
@@ -253,9 +255,12 @@ async def find_actor(
         for addr in registry_addrs
     )
     portals: list[Portal]
-    async with gather_contexts(
-        mngrs=maybe_portals,
-    ) as portals:
+    async with (
+        collapse_eg(),
+        gather_contexts(
+            mngrs=maybe_portals,
+        ) as portals,
+    ):
         # log.runtime(
         #     'Gathered portals:\n'
         #     f'{portals}'

tractor/_entry.py

@@ -21,8 +21,7 @@ Sub-process entry points.
 from __future__ import annotations
 from functools import partial
 import multiprocessing as mp
-import os
-import textwrap
+# import os
 from typing import (
     Any,
     TYPE_CHECKING,
@@ -35,7 +34,11 @@ from .log import (
     get_logger,
 )
 from . import _state
-from .devx import _debug
+from .devx import (
+    _frame_stack,
+    pformat,
+)
+# from .msg import pretty_struct
 from .to_asyncio import run_as_asyncio_guest
 from ._addr import UnwrappedAddress
 from ._runtime import (
@@ -103,107 +106,6 @@ def _mp_main(
         )
 
 
-# TODO: move this func to some kinda `.devx._conc_lang.py` eventually
-# as we work out our multi-domain state-flow-syntax!
-def nest_from_op(
-    input_op: str,
-    #
-    # ?TODO? an idea for a syntax to the state of concurrent systems
-    # as a "3-domain" (execution, scope, storage) model and using
-    # a minimal ascii/utf-8 operator-set.
-    #
-    # try not to take any of this seriously yet XD
-    #
-    # > is a "play operator" indicating (CPU bound)
-    #   exec/work/ops required at the "lowest level computing"
-    #
-    # execution primititves (tasks, threads, actors..) denote their
-    # lifetime with '(' and ')' since parentheses normally are used
-    # in many langs to denote function calls.
-    #
-    # starting = (
-    # >(  opening/starting; beginning of the thread-of-exec (toe?)
-    # (>  opened/started,  (finished spawning toe)
-    # |_<Task: blah blah..>  repr of toe, in py these look like <objs>
-    #
-    # >) closing/exiting/stopping,
-    # )> closed/exited/stopped,
-    # |_<Task: blah blah..>
-    #   [OR <), )< ?? ]
-    #
-    # ending = )
-    # >c) cancelling to close/exit
-    # c)> cancelled (caused close), OR?
-    #  |_<Actor: ..>
-    #   OR maybe "<c)" which better indicates the cancel being
-    #   "delivered/returned" / returned" to LHS?
-    #
-    # >x)  erroring to eventuall exit
-    # x)>  errored and terminated
-    #  |_<Actor: ...>
-    #
-    # scopes: supers/nurseries, IPC-ctxs, sessions, perms, etc.
-    # >{  opening
-    # {>  opened
-    # }>  closed
-    # >}  closing
-    #
-    # storage: like queues, shm-buffers, files, etc..
-    # >[  opening
-    # [>  opened
-    #  |_<FileObj: ..>
-    #
-    # >]  closing
-    # ]>  closed
-
-    # IPC ops: channels, transports, msging
-    # =>  req msg
-    # <=  resp msg
-    # <=> 2-way streaming (of msgs)
-    # <-  recv 1 msg
-    # ->  send 1 msg
-    #
-    # TODO: still not sure on R/L-HS approach..?
-    # =>(  send-req to exec start (task, actor, thread..)
-    # (<=  recv-req to ^
-    #
-    # (<=  recv-req ^
-    # <=(  recv-resp opened remote exec primitive
-    # <=)  recv-resp closed
-    #
-    # )<=c req to stop due to cancel
-    # c=>) req to stop due to cancel
-    #
-    # =>{  recv-req to open
-    # <={  send-status that it closed
-
-    tree_str: str,
-
-    # NOTE: so move back-from-the-left of the `input_op` by
-    # this amount.
-    back_from_op: int = 0,
-) -> str:
-    '''
-    Depth-increment the input (presumably hierarchy/supervision)
-    input "tree string" below the provided `input_op` execution
-    operator, so injecting a `"\n|_{input_op}\n"`and indenting the
-    `tree_str` to nest content aligned with the ops last char.
-
-    '''
-    return (
-        f'{input_op}\n'
-        +
-        textwrap.indent(
-            tree_str,
-            prefix=(
-                len(input_op)
-                -
-                (back_from_op + 1)
-            ) * ' ',
-        )
-    )
-
-
 def _trio_main(
     actor: Actor,
     *,
@@ -215,7 +117,7 @@ def _trio_main(
     Entry point for a `trio_run_in_process` subactor.
 
     '''
-    _debug.hide_runtime_frames()
+    _frame_stack.hide_runtime_frames()
 
     _state._current_actor = actor
     trio_main = partial(
@@ -226,30 +128,23 @@ def _trio_main(
 
     if actor.loglevel is not None:
         get_console_log(actor.loglevel)
-        actor_info: str = (
-            f'|_{actor}\n'
-            f'  uid: {actor.uid}\n'
-            f'  pid: {os.getpid()}\n'
-            f'  parent_addr: {parent_addr}\n'
-            f'  loglevel: {actor.loglevel}\n'
-        )
         log.info(
-            'Starting new `trio` subactor:\n'
+            f'Starting `trio` subactor from parent @ '
+            f'{parent_addr}\n'
             +
-            nest_from_op(
+            pformat.nest_from_op(
                 input_op='>(',  # see syntax ideas above
-                tree_str=actor_info,
-                back_from_op=2,  # since "complete"
+                text=f'{actor}',
             )
         )
     logmeth = log.info
     exit_status: str = (
         'Subactor exited\n'
         +
-        nest_from_op(
+        pformat.nest_from_op(
             input_op=')>',  # like a "closed-to-play"-icon from super perspective
-            tree_str=actor_info,
-            back_from_op=1,
+            text=f'{actor}',
+            nest_indent=1,
         )
     )
     try:
@@ -264,9 +159,9 @@ def _trio_main(
         exit_status: str = (
             'Actor received KBI (aka an OS-cancel)\n'
             +
-            nest_from_op(
+            pformat.nest_from_op(
                 input_op='c)>',  # closed due to cancel (see above)
-                tree_str=actor_info,
+                text=f'{actor}',
             )
         )
     except BaseException as err:
@@ -274,9 +169,9 @@ def _trio_main(
         exit_status: str = (
             'Main actor task exited due to crash?\n'
             +
-            nest_from_op(
+            pformat.nest_from_op(
                 input_op='x)>',  # closed by error
-                tree_str=actor_info,
+                text=f'{actor}',
             )
         )
         # NOTE since we raise a tb will already be shown on the

tractor/_exceptions.py

@@ -1246,55 +1246,6 @@ def unpack_error(
     return exc
 
 
-def is_multi_cancelled(
-    exc: BaseException|BaseExceptionGroup,
-
-    ignore_nested: set[BaseException] = set(),
-
-) -> bool|BaseExceptionGroup:
-    '''
-    Predicate to determine if an `BaseExceptionGroup` only contains
-    some (maybe nested) set of sub-grouped exceptions (like only
-    `trio.Cancelled`s which get swallowed silently by default) and is
-    thus the result of "gracefully cancelling" a collection of
-    sub-tasks (or other conc primitives) and receiving a "cancelled
-    ACK" from each after termination.
-
-    Docs:
-    ----
-    - https://docs.python.org/3/library/exceptions.html#exception-groups
-    - https://docs.python.org/3/library/exceptions.html#BaseExceptionGroup.subgroup
-
-    '''
-
-    if (
-        not ignore_nested
-        or
-        trio.Cancelled in ignore_nested
-        # XXX always count-in `trio`'s native signal
-    ):
-        ignore_nested.update({trio.Cancelled})
-
-    if isinstance(exc, BaseExceptionGroup):
-        matched_exc: BaseExceptionGroup|None = exc.subgroup(
-            tuple(ignore_nested),
-
-            # TODO, complain about why not allowed XD
-            # condition=tuple(ignore_nested),
-        )
-        if matched_exc is not None:
-            return matched_exc
-
-    # NOTE, IFF no excs types match (throughout the error-tree)
-    # -> return `False`, OW return the matched sub-eg.
-    #
-    # IOW, for the inverse of ^ for the purpose of
-    # maybe-enter-REPL--logic: "only debug when the err-tree contains
-    # at least one exc-type NOT in `ignore_nested`" ; i.e. the case where
-    # we fallthrough and return `False` here.
-    return False
-
-
 def _raise_from_unexpected_msg(
     ctx: Context,
     msg: MsgType,

tractor/_portal.py

@@ -39,7 +39,10 @@ import warnings
 
 import trio
 
-from .trionics import maybe_open_nursery
+from .trionics import (
+    maybe_open_nursery,
+    collapse_eg,
+)
 from ._state import (
     current_actor,
 )
@@ -115,6 +118,10 @@ class Portal:
 
     @property
     def chan(self) -> Channel:
+        '''
+        Ref to this ctx's underlying `tractor.ipc.Channel`.
+
+        '''
         return self._chan
 
     @property
@@ -174,10 +181,17 @@ class Portal:
 
         # not expecting a "main" result
         if self._expect_result_ctx is None:
+            peer_id: str = f'{self.channel.aid.reprol()!r}'
             log.warning(
-                f"Portal for {self.channel.aid} not expecting a final"
-                " result?\nresult() should only be called if subactor"
-                " was spawned with `ActorNursery.run_in_actor()`")
+                f'Portal to peer {peer_id} will not deliver a final result?\n'
+                f'\n'
+                f'Context.result() can only be called by the parent of '
+                f'a sub-actor when it was spawned with '
+                f'`ActorNursery.run_in_actor()`'
+                f'\n'
+                f'Further this `ActorNursery`-method-API will deprecated in the'
+                f'near fututre!\n'
+            )
             return NoResult
 
         # expecting a "main" result
@@ -210,6 +224,7 @@ class Portal:
         typname: str = type(self).__name__
         log.warning(
             f'`{typname}.result()` is DEPRECATED!\n'
+            f'\n'
             f'Use `{typname}.wait_for_result()` instead!\n'
         )
         return await self.wait_for_result(
@@ -221,8 +236,10 @@ class Portal:
         # terminate all locally running async generator
         # IPC calls
         if self._streams:
-            log.cancel(
-                f"Cancelling all streams with {self.channel.aid}")
+            peer_id: str = f'{self.channel.aid.reprol()!r}'
+            report: str = (
+                f'Cancelling all msg-streams with {peer_id}\n'
+            )
             for stream in self._streams.copy():
                 try:
                     await stream.aclose()
@@ -231,10 +248,18 @@ class Portal:
                     # (unless of course at some point down the road we
                     # won't expect this to always be the case or need to
                     # detect it for respawning purposes?)
-                    log.debug(f"{stream} was already closed.")
+                    report += (
+                        f'->) {stream!r} already closed\n'
+                    )
+
+            log.cancel(report)
 
     async def aclose(self):
-        log.debug(f"Closing {self}")
+        log.debug(
+            f'Closing portal\n'
+            f'>}}\n'
+            f'|_{self}\n'
+        )
         # TODO: once we move to implementing our own `ReceiveChannel`
         # (including remote task cancellation inside its `.aclose()`)
         # we'll need to .aclose all those channels here
@@ -260,23 +285,22 @@ class Portal:
         __runtimeframe__: int = 1  # noqa
 
         chan: Channel = self.channel
+        peer_id: str = f'{self.channel.aid.reprol()!r}'
         if not chan.connected():
             log.runtime(
-                'This channel is already closed, skipping cancel request..'
+                'Peer {peer_id} is already disconnected\n'
+                '-> skipping cancel request..\n'
             )
             return False
 
-        reminfo: str = (
-            f'c)=> {self.channel.aid}\n'
-            f'  |_{chan}\n'
-        )
         log.cancel(
-            f'Requesting actor-runtime cancel for peer\n\n'
-            f'{reminfo}'
+            f'Sending actor-runtime-cancel-req to peer\n'
+            f'\n'
+            f'c)=> {peer_id}\n'
         )
 
         # XXX the one spot we set it?
-        self.channel._cancel_called: bool = True
+        chan._cancel_called: bool = True
         try:
             # send cancel cmd - might not get response
             # XXX: sure would be nice to make this work with
@@ -297,8 +321,9 @@ class Portal:
                 # may timeout and we never get an ack (obvi racy)
                 # but that doesn't mean it wasn't cancelled.
                 log.debug(
-                    'May have failed to cancel peer?\n'
-                    f'{reminfo}'
+                    f'May have failed to cancel peer?\n'
+                    f'\n'
+                    f'c)=?> {peer_id}\n'
                 )
 
             # if we get here some weird cancellation case happened
@@ -316,22 +341,22 @@ class Portal:
 
             TransportClosed,
         ) as tpt_err:
-            report: str = (
-                f'IPC chan for actor already closed or broken?\n\n'
-                f'{self.channel.aid}\n'
-                f' |_{self.channel}\n'
+            ipc_borked_report: str = (
+                f'IPC for actor already closed/broken?\n\n'
+                f'\n'
+                f'c)=x> {peer_id}\n'
             )
             match tpt_err:
                 case TransportClosed():
-                    log.debug(report)
+                    log.debug(ipc_borked_report)
                 case _:
-                    report += (
+                    ipc_borked_report += (
                         f'\n'
                         f'Unhandled low-level transport-closed/error during\n'
                         f'Portal.cancel_actor()` request?\n'
                         f'<{type(tpt_err).__name__}( {tpt_err} )>\n'
                     )
-                    log.warning(report)
+                    log.warning(ipc_borked_report)
 
             return False
 
@@ -488,10 +513,13 @@ class Portal:
                 with trio.CancelScope(shield=True):
                     await ctx.cancel()
 
-            except trio.ClosedResourceError:
+            except trio.ClosedResourceError as cre:
                 # if the far end terminates before we send a cancel the
                 # underlying transport-channel may already be closed.
-                log.cancel(f'Context {ctx} was already closed?')
+                log.cancel(
+                    f'Context.cancel() -> {cre!r}\n'
+                    f'cid: {ctx.cid!r} already closed?\n'
+                )
 
             # XXX: should this always be done?
             # await recv_chan.aclose()
@@ -558,14 +586,13 @@ async def open_portal(
     assert actor
     was_connected: bool = False
 
-    async with maybe_open_nursery(
-        tn,
-        shield=shield,
-        strict_exception_groups=False,
-        # ^XXX^ TODO? soo roll our own then ??
-        # -> since we kinda want the "if only one `.exception` then
-        # just raise that" interface?
-    ) as tn:
+    async with (
+        collapse_eg(),
+        maybe_open_nursery(
+            tn,
+            shield=shield,
+        ) as tn,
+    ):
 
         if not channel.connected():
             await channel.connect()

tractor/_root.py

@@ -37,14 +37,12 @@ import warnings
 
 import trio
 
-from ._runtime import (
-    Actor,
-    Arbiter,
-    # TODO: rename and make a non-actor subtype?
-    # Arbiter as Registry,
-    async_main,
+from . import _runtime
+from .devx import (
+    debug,
+    _frame_stack,
+    pformat as _pformat,
 )
-from .devx import _debug
 from . import _spawn
 from . import _state
 from . import log
@@ -58,16 +56,19 @@ from ._addr import (
     mk_uuid,
     wrap_address,
 )
+from .trionics import (
+    is_multi_cancelled,
+    collapse_eg,
+)
 from ._exceptions import (
     RuntimeFailure,
-    is_multi_cancelled,
 )
 
 
 logger = log.get_logger('tractor')
 
 
-# TODO: stick this in a `@acm` defined in `devx._debug`?
+# TODO: stick this in a `@acm` defined in `devx.debug`?
 # -[ ] also maybe consider making this a `wrapt`-deco to
 #     save an indent level?
 #
@@ -89,17 +90,17 @@ async def maybe_block_bp(
         debug_mode
         and maybe_enable_greenback
         and (
-            maybe_mod := await _debug.maybe_init_greenback(
+            maybe_mod := await debug.maybe_init_greenback(
                 raise_not_found=False,
             )
         )
     ):
         logger.info(
             f'Found `greenback` installed @ {maybe_mod}\n'
-            'Enabling `tractor.pause_from_sync()` support!\n'
+            f'Enabling `tractor.pause_from_sync()` support!\n'
         )
         os.environ['PYTHONBREAKPOINT'] = (
-            'tractor.devx._debug._sync_pause_from_builtin'
+            'tractor.devx.debug._sync_pause_from_builtin'
         )
         _state._runtime_vars['use_greenback'] = True
         bp_blocked = False
@@ -163,7 +164,9 @@ async def open_root_actor(
 
     # enables the multi-process debugger support
     debug_mode: bool = False,
-    maybe_enable_greenback: bool = True,  # `.pause_from_sync()/breakpoint()` support
+    maybe_enable_greenback: bool = False,  # `.pause_from_sync()/breakpoint()` support
+    # ^XXX NOTE^ the perf implications of use,
+    # https://greenback.readthedocs.io/en/latest/principle.html#performance
     enable_stack_on_sig: bool = False,
 
     # internal logging
@@ -178,7 +181,7 @@ async def open_root_actor(
 
     hide_tb: bool = True,
 
-    # XXX, proxied directly to `.devx._debug._maybe_enter_pm()`
+    # XXX, proxied directly to `.devx.debug._maybe_enter_pm()`
     # for REPL-entry logic.
     debug_filter: Callable[
         [BaseException|BaseExceptionGroup],
@@ -189,13 +192,19 @@ async def open_root_actor(
     # read-only state to sublayers?
     # extra_rt_vars: dict|None = None,
 
-) -> Actor:
+) -> _runtime.Actor:
     '''
-    Runtime init entry point for ``tractor``.
+    Initialize the `tractor` runtime by starting a "root actor" in
+    a parent-most Python process.
+
+    All (disjoint) actor-process-trees-as-programs are created via
+    this entrypoint.
 
     '''
     # XXX NEVER allow nested actor-trees!
-    if already_actor := _state.current_actor(err_on_no_runtime=False):
+    if already_actor := _state.current_actor(
+        err_on_no_runtime=False,
+    ):
         rtvs: dict[str, Any] = _state._runtime_vars
         root_mailbox: list[str, int] = rtvs['_root_mailbox']
         registry_addrs: list[list[str, int]] = rtvs['_registry_addrs']
@@ -228,12 +237,12 @@ async def open_root_actor(
                 f'enable_transports={enable_transports!r}\n'
             )
 
-        _debug.hide_runtime_frames()
+        _frame_stack.hide_runtime_frames()
         __tracebackhide__: bool = hide_tb
 
         # attempt to retreive ``trio``'s sigint handler and stash it
         # on our debugger lock state.
-        _debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
+        debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
 
         # mark top most level process as root actor
         _state._runtime_vars['_is_root'] = True
@@ -265,14 +274,20 @@ async def open_root_actor(
                 DeprecationWarning,
                 stacklevel=2,
             )
-            registry_addrs = [arbiter_addr]
+            uw_reg_addrs = [arbiter_addr]
 
-        if not registry_addrs:
-            registry_addrs: list[UnwrappedAddress] = default_lo_addrs(
+        uw_reg_addrs = registry_addrs
+        if not uw_reg_addrs:
+            uw_reg_addrs: list[UnwrappedAddress] = default_lo_addrs(
                 enable_transports
             )
 
-        assert registry_addrs
+        # must exist by now since all below code is dependent
+        assert uw_reg_addrs
+        registry_addrs: list[Address] = [
+            wrap_address(uw_addr)
+            for uw_addr in uw_reg_addrs
+        ]
 
         loglevel = (
             loglevel
@@ -281,13 +296,14 @@ async def open_root_actor(
 
         if (
             debug_mode
-            and _spawn._spawn_method == 'trio'
+            and
+            _spawn._spawn_method == 'trio'
         ):
             _state._runtime_vars['_debug_mode'] = True
 
             # expose internal debug module to every actor allowing for
             # use of ``await tractor.pause()``
-            enable_modules.append('tractor.devx._debug')
+            enable_modules.append('tractor.devx.debug._tty_lock')
 
             # if debug mode get's enabled *at least* use that level of
             # logging for some informative console prompts.
@@ -320,10 +336,10 @@ async def open_root_actor(
             enable_stack_on_sig()
 
         # closed into below ping task-func
-        ponged_addrs: list[UnwrappedAddress] = []
+        ponged_addrs: list[Address] = []
 
         async def ping_tpt_socket(
-            addr: UnwrappedAddress,
+            addr: Address,
             timeout: float = 1,
         ) -> None:
             '''
@@ -343,17 +359,22 @@ async def open_root_actor(
                 # be better to eventually have a "discovery" protocol
                 # with basic handshake instead?
                 with trio.move_on_after(timeout):
-                    async with _connect_chan(addr):
+                    async with _connect_chan(addr.unwrap()):
                         ponged_addrs.append(addr)
 
             except OSError:
-                # TODO: make this a "discovery" log level?
+                # ?TODO, make this a "discovery" log level?
                 logger.info(
-                    f'No actor registry found @ {addr}\n'
+                    f'No root-actor registry found @ {addr!r}\n'
                 )
 
+        # !TODO, this is basically just another (abstract)
+        # happy-eyeballs, so we should try for formalize it somewhere
+        # in a `.[_]discovery` ya?
+        #
         async with trio.open_nursery() as tn:
-            for addr in registry_addrs:
+            for uw_addr in uw_reg_addrs:
+                addr: Address = wrap_address(uw_addr)
                 tn.start_soon(
                     ping_tpt_socket,
                     addr,
@@ -375,31 +396,35 @@ async def open_root_actor(
                 f'Registry(s) seem(s) to exist @ {ponged_addrs}'
             )
 
-            actor = Actor(
+            actor = _runtime.Actor(
                 name=name or 'anonymous',
                 uuid=mk_uuid(),
                 registry_addrs=ponged_addrs,
                 loglevel=loglevel,
                 enable_modules=enable_modules,
             )
-            # DO NOT use the registry_addrs as the transport server
-            # addrs for this new non-registar, root-actor.
+            # **DO NOT** use the registry_addrs as the
+            # ipc-transport-server's bind-addrs as this is
+            # a new NON-registrar, ROOT-actor.
+            #
+            # XXX INSTEAD, bind random addrs using the same tpt
+            # proto.
             for addr in ponged_addrs:
-                waddr: Address = wrap_address(addr)
                 trans_bind_addrs.append(
-                    waddr.get_random(bindspace=waddr.bindspace)
+                    addr.get_random(
+                        bindspace=addr.bindspace,
+                    )
                 )
 
         # Start this local actor as the "registrar", aka a regular
         # actor who manages the local registry of "mailboxes" of
         # other process-tree-local sub-actors.
         else:
-
             # NOTE that if the current actor IS THE REGISTAR, the
             # following init steps are taken:
             # - the tranport layer server is bound to each addr
             #   pair defined in provided registry_addrs, or the default.
-            trans_bind_addrs = registry_addrs
+            trans_bind_addrs = uw_reg_addrs
 
             # - it is normally desirable for any registrar to stay up
             #   indefinitely until either all registered (child/sub)
@@ -410,7 +435,8 @@ async def open_root_actor(
             # https://github.com/goodboy/tractor/pull/348
             # https://github.com/goodboy/tractor/issues/296
 
-            actor = Arbiter(
+            # TODO: rename as `RootActor` or is that even necessary?
+            actor = _runtime.Arbiter(
                 name=name or 'registrar',
                 uuid=mk_uuid(),
                 registry_addrs=registry_addrs,
@@ -422,6 +448,16 @@ async def open_root_actor(
             # `.trio.run()`.
             actor._infected_aio = _state._runtime_vars['_is_infected_aio']
 
+        # NOTE, only set the loopback addr for the
+        # process-tree-global "root" mailbox since all sub-actors
+        # should be able to speak to their root actor over that
+        # channel.
+        raddrs: list[Address] = _state._runtime_vars['_root_addrs']
+        raddrs.extend(trans_bind_addrs)
+        # TODO, remove once we have also removed all usage;
+        # eventually all (root-)registry apis should expect > 1 addr.
+        _state._runtime_vars['_root_mailbox'] = raddrs[0]
+
         # Start up main task set via core actor-runtime nurseries.
         try:
             # assign process-local actor
@@ -429,21 +465,28 @@ async def open_root_actor(
 
             # start local channel-server and fake the portal API
             # NOTE: this won't block since we provide the nursery
-            ml_addrs_str: str = '\n'.join(
-                f'@{addr}' for addr in trans_bind_addrs
-            )
-            logger.info(
-                f'Starting local {actor.uid} on the following transport addrs:\n'
-                f'{ml_addrs_str}'
-            )
+            report: str = f'Starting actor-runtime for {actor.aid.reprol()!r}\n'
+            if reg_addrs := actor.registry_addrs:
+                report += (
+                    '-> Opening new registry @ '
+                    +
+                    '\n'.join(
+                        f'{addr}' for addr in reg_addrs
+                    )
+                )
+            logger.info(f'{report}\n')
 
-            # start the actor runtime in a new task
-            async with trio.open_nursery(
-                strict_exception_groups=False,
-                # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
-            ) as nursery:
+            # start runtime in a bg sub-task, yield to caller.
+            async with (
+                collapse_eg(),
+                trio.open_nursery() as root_tn,
 
-                # ``_runtime.async_main()`` creates an internal nursery
+                # ?TODO? finally-footgun below?
+                # -> see note on why shielding.
+                # maybe_raise_from_masking_exc(),
+            ):
+                actor._root_tn = root_tn
+                # `_runtime.async_main()` creates an internal nursery
                 # and blocks here until any underlying actor(-process)
                 # tree has terminated thereby conducting so called
                 # "end-to-end" structured concurrency throughout an
@@ -451,9 +494,9 @@ async def open_root_actor(
                 # "actor runtime" primitives are SC-compat and thus all
                 # transitively spawned actors/processes must be as
                 # well.
-                await nursery.start(
+                await root_tn.start(
                     partial(
-                        async_main,
+                        _runtime.async_main,
                         actor,
                         accept_addrs=trans_bind_addrs,
                         parent_addr=None
@@ -469,7 +512,7 @@ async def open_root_actor(
                     # TODO, in beginning to handle the subsubactor with
                     # crashed grandparent cases..
                     #
-                    # was_locked: bool = await _debug.maybe_wait_for_debugger(
+                    # was_locked: bool = await debug.maybe_wait_for_debugger(
                     #     child_in_debug=True,
                     # )
                     # XXX NOTE XXX see equiv note inside
@@ -477,10 +520,15 @@ async def open_root_actor(
                     # non-root or root-that-opened-this-mahually case we
                     # wait for the local actor-nursery to exit before
                     # exiting the transport channel handler.
-                    entered: bool = await _debug._maybe_enter_pm(
+                    entered: bool = await debug._maybe_enter_pm(
                         err,
                         api_frame=inspect.currentframe(),
                         debug_filter=debug_filter,
+
+                        # XXX NOTE, required to debug root-actor
+                        # crashes under cancellation conditions; so
+                        # most of them!
+                        shield=root_tn.cancel_scope.cancel_called,
                     )
 
                     if (
@@ -501,7 +549,7 @@ async def open_root_actor(
                     raise
 
                 finally:
-                    # NOTE: not sure if we'll ever need this but it's
+                    # NOTE/TODO?, not sure if we'll ever need this but it's
                     # possibly better for even more determinism?
                     # logger.cancel(
                     #     f'Waiting on {len(nurseries)} nurseries in root..')
@@ -510,19 +558,44 @@ async def open_root_actor(
                     #     for an in nurseries:
                     #         tempn.start_soon(an.exited.wait)
 
+                    op_nested_actor_repr: str = _pformat.nest_from_op(
+                        input_op='>) ',
+                        text=actor.pformat(),
+                        nest_prefix='|_',
+                    )
                     logger.info(
                         f'Closing down root actor\n'
-                        f'>)\n'
-                        f'|_{actor}\n'
+                        f'{op_nested_actor_repr}'
                     )
-                    await actor.cancel(None)  # self cancel
+                    # XXX, THIS IS A *finally-footgun*!
+                    # (also mentioned in with-block above)
+                    # -> though already shields iternally it can
+                    # taskc here and mask underlying errors raised in
+                    # the try-block above?
+                    with trio.CancelScope(shield=True):
+                        await actor.cancel(None)  # self cancel
         finally:
+            # revert all process-global runtime state
+            if (
+                debug_mode
+                and
+                _spawn._spawn_method == 'trio'
+            ):
+                _state._runtime_vars['_debug_mode'] = False
+
             _state._current_actor = None
             _state._last_actor_terminated = actor
-            logger.runtime(
+
+            sclang_repr: str = _pformat.nest_from_op(
+                input_op=')>',
+                text=actor.pformat(),
+                nest_prefix='|_',
+                nest_indent=1,
+            )
+
+            logger.info(
                 f'Root actor terminated\n'
-                f')>\n'
-                f' |_{actor}\n'
+                f'{sclang_repr}'
             )
 
 

tractor/_rpc.py

@@ -37,6 +37,7 @@ import warnings
 
 import trio
 from trio import (
+    Cancelled,
     CancelScope,
     Nursery,
     TaskStatus,
@@ -52,13 +53,18 @@ from ._exceptions import (
     ModuleNotExposed,
     MsgTypeError,
     TransportClosed,
-    is_multi_cancelled,
     pack_error,
     unpack_error,
 )
+from .trionics import (
+    collapse_eg,
+    is_multi_cancelled,
+    maybe_raise_from_masking_exc,
+)
 from .devx import (
-    _debug,
+    debug,
     add_div,
+    pformat as _pformat,
 )
 from . import _state
 from .log import get_logger
@@ -67,7 +73,7 @@ from .msg import (
     MsgCodec,
     PayloadT,
     NamespacePath,
-    # pretty_struct,
+    pretty_struct,
     _ops as msgops,
 )
 from tractor.msg.types import (
@@ -215,11 +221,18 @@ async def _invoke_non_context(
             task_status.started(ctx)
             result = await coro
             fname: str = func.__name__
+
+            op_nested_task: str = _pformat.nest_from_op(
+                input_op=f')> cid: {ctx.cid!r}',
+                text=f'{ctx._task}',
+                nest_indent=1,  # under >
+            )
             log.runtime(
-                'RPC complete:\n'
-                f'task: {ctx._task}\n'
-                f'|_cid={ctx.cid}\n'
-                f'|_{fname}() -> {pformat(result)}\n'
+                f'RPC task complete\n'
+                f'\n'
+                f'{op_nested_task}\n'
+                f'\n'
+                f')> {fname}() -> {pformat(result)}\n'
             )
 
             # NOTE: only send result if we know IPC isn't down
@@ -250,7 +263,7 @@ async def _errors_relayed_via_ipc(
     ctx: Context,
     is_rpc: bool,
 
-    hide_tb: bool = False,
+    hide_tb: bool = True,
     debug_kbis: bool = False,
     task_status: TaskStatus[
         Context | BaseException
@@ -266,7 +279,7 @@ async def _errors_relayed_via_ipc(
 
     # TODO: a debug nursery when in debug mode!
     # async with maybe_open_debugger_nursery() as debug_tn:
-    # => see matching comment in side `._debug._pause()`
+    # => see matching comment in side `.debug._pause()`
     rpc_err: BaseException|None = None
     try:
         yield  # run RPC invoke body
@@ -318,7 +331,7 @@ async def _errors_relayed_via_ipc(
                     'RPC task crashed, attempting to enter debugger\n'
                     f'|_{ctx}'
                 )
-                entered_debug = await _debug._maybe_enter_pm(
+                entered_debug = await debug._maybe_enter_pm(
                     err,
                     api_frame=inspect.currentframe(),
                 )
@@ -371,13 +384,13 @@ async def _errors_relayed_via_ipc(
 
     # RPC task bookeeping.
     # since RPC tasks are scheduled inside a flat
-    # `Actor._service_n`, we add "handles" to each such that
+    # `Actor._service_tn`, we add "handles" to each such that
     # they can be individually ccancelled.
     finally:
 
-        # if the error is not from user code and instead a failure
-        # of a runtime RPC or transport failure we do prolly want to
-        # show this frame
+        # if the error is not from user code and instead a failure of
+        # an internal-runtime-RPC or IPC-connection, we do (prolly) want
+        # to show this frame!
         if (
             rpc_err
             and (
@@ -449,7 +462,7 @@ async def _invoke(
     connected IPC channel.
 
     This is the core "RPC" `trio.Task` scheduling machinery used to start every
-    remotely invoked function, normally in `Actor._service_n: Nursery`.
+    remotely invoked function, normally in `Actor._service_tn: Nursery`.
 
     '''
     __tracebackhide__: bool = hide_tb
@@ -462,7 +475,7 @@ async def _invoke(
     ):
         # XXX for .pause_from_sync()` usage we need to make sure
         # `greenback` is boostrapped in the subactor!
-        await _debug.maybe_init_greenback()
+        await debug.maybe_init_greenback()
 
     # TODO: possibly a specially formatted traceback
     # (not sure what typing is for this..)?
@@ -616,32 +629,39 @@ async def _invoke(
         #  -> the below scope is never exposed to the
         #     `@context` marked RPC function.
         # - `._portal` is never set.
+        scope_err: BaseException|None = None
         try:
-            tn: trio.Nursery
+            # TODO: better `trionics` primitive/tooling usage here!
+            # -[ ] should would be nice to have our `TaskMngr`
+            #   nursery here!
+            # -[ ] payload value checking like we do with
+            #   `.started()` such that the debbuger can engage
+            #   here in the child task instead of waiting for the
+            #   parent to crash with it's own MTE..
+            #
+            tn: Nursery
             rpc_ctx_cs: CancelScope
             async with (
-                trio.open_nursery(
-                    strict_exception_groups=False,
-                    # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
-
-                ) as tn,
+                collapse_eg(hide_tb=False),
+                trio.open_nursery() as tn,
                 msgops.maybe_limit_plds(
                     ctx=ctx,
                     spec=ctx_meta.get('pld_spec'),
                     dec_hook=ctx_meta.get('dec_hook'),
                 ),
+
+                # XXX NOTE, this being the "most embedded"
+                # scope ensures unasking of the `await coro` below
+                # *should* never be interfered with!!
+                maybe_raise_from_masking_exc(
+                    unmask_from=(Cancelled,),
+                ) as _mbme,  # maybe boxed masked exc
             ):
                 ctx._scope_nursery = tn
                 rpc_ctx_cs = ctx._scope = tn.cancel_scope
                 task_status.started(ctx)
 
-                # TODO: better `trionics` tooling:
-                # -[ ] should would be nice to have our `TaskMngr`
-                #   nursery here!
-                # -[ ] payload value checking like we do with
-                #   `.started()` such that the debbuger can engage
-                #   here in the child task instead of waiting for the
-                #   parent to crash with it's own MTE..
+                # invoke user endpoint fn.
                 res: Any|PayloadT = await coro
                 return_msg: Return|CancelAck = return_msg_type(
                     cid=cid,
@@ -651,7 +671,8 @@ async def _invoke(
                 ctx._result = res
                 log.runtime(
                     f'Sending result msg and exiting {ctx.side!r}\n'
-                    f'{return_msg}\n'
+                    f'\n'
+                    f'{pretty_struct.pformat(return_msg)}\n'
                 )
                 await chan.send(return_msg)
 
@@ -743,43 +764,52 @@ async def _invoke(
             BaseExceptionGroup,
             BaseException,
             trio.Cancelled,
-
-        ) as scope_error:
+        ) as _scope_err:
+            scope_err = _scope_err
             if (
-                isinstance(scope_error, RuntimeError)
-                and scope_error.args
-                and 'Cancel scope stack corrupted' in scope_error.args[0]
+                isinstance(scope_err, RuntimeError)
+                and
+                scope_err.args
+                and
+                'Cancel scope stack corrupted' in scope_err.args[0]
             ):
                 log.exception('Cancel scope stack corrupted!?\n')
-                # _debug.mk_pdb().set_trace()
+                # debug.mk_pdb().set_trace()
 
             # always set this (child) side's exception as the
             # local error on the context
-            ctx._local_error: BaseException = scope_error
+            ctx._local_error: BaseException = scope_err
             # ^-TODO-^ question,
             # does this matter other then for
             # consistentcy/testing?
             # |_ no user code should be in this scope at this point
             #    AND we already set this in the block below?
 
-            # if a remote error was set then likely the
-            # exception group was raised due to that, so
+            # XXX if a remote error was set then likely the
+            # exc group was raised due to that, so
             # and we instead raise that error immediately!
-            ctx.maybe_raise()
+            maybe_re: (
+                ContextCancelled|RemoteActorError
+            ) = ctx.maybe_raise()
+            if maybe_re:
+                log.cancel(
+                    f'Suppressing remote-exc from peer,\n'
+                    f'{maybe_re!r}\n'
+                )
 
             # maybe TODO: pack in come kinda
             # `trio.Cancelled.__traceback__` here so they can be
             # unwrapped and displayed on the caller side? no se..
-            raise
+            raise scope_err
 
         # `@context` entrypoint task bookeeping.
         # i.e. only pop the context tracking if used ;)
         finally:
-            assert chan.uid
+            assert chan.aid
 
             # don't pop the local context until we know the
             # associated child isn't in debug any more
-            await _debug.maybe_wait_for_debugger()
+            await debug.maybe_wait_for_debugger()
             ctx: Context = actor._contexts.pop((
                 chan.uid,
                 cid,
@@ -792,26 +822,49 @@ async def _invoke(
                 f'after having {ctx.repr_state!r}\n'
             )
             if merr:
-
                 logmeth: Callable = log.error
-                if isinstance(merr, ContextCancelled):
-                    logmeth: Callable = log.runtime
+                if (
+                    # ctxc: by `Context.cancel()`
+                    isinstance(merr, ContextCancelled)
 
-                if not isinstance(merr, RemoteActorError):
-                    tb_str: str = ''.join(traceback.format_exception(merr))
+                    # out-of-layer cancellation, one of:
+                    # - actorc: by `Portal.cancel_actor()`
+                    # - OSc: by SIGINT or `Process.signal()`
+                    or (
+                        isinstance(merr, trio.Cancelled)
+                        and
+                        ctx.canceller
+                    )
+                ):
+                    logmeth: Callable = log.cancel
+                    descr_str += (
+                        f' with {merr!r}\n'
+                    )
+
+                elif (
+                    not isinstance(merr, RemoteActorError)
+                ):
+                    tb_str: str = ''.join(
+                        traceback.format_exception(merr)
+                    )
                     descr_str += (
                         f'\n{merr!r}\n'  # needed?
                         f'{tb_str}\n'
                     )
                 else:
-                    descr_str += f'\n{merr!r}\n'
+                    descr_str += (
+                        f'{merr!r}\n'
+                    )
             else:
-                descr_str += f'\nand final result {ctx.outcome!r}\n'
+                descr_str += (
+                    f'\n'
+                    f'with final result {ctx.outcome!r}\n'
+                )
 
             logmeth(
-                message
-                +
-                descr_str
+                f'{message}\n'
+                f'\n'
+                f'{descr_str}\n'
             )
 
 
@@ -882,7 +935,7 @@ async def process_messages(
 
     Receive (multiplexed) per-`Channel` RPC requests as msgs from
     remote processes; schedule target async funcs as local
-    `trio.Task`s inside the `Actor._service_n: Nursery`.
+    `trio.Task`s inside the `Actor._service_tn: Nursery`.
 
     Depending on msg type, non-`cmd` (task spawning/starting)
     request payloads (eg. `started`, `yield`, `return`, `error`)
@@ -907,7 +960,7 @@ async def process_messages(
 
     '''
     actor: Actor = _state.current_actor()
-    assert actor._service_n  # runtime state sanity
+    assert actor._service_tn  # runtime state sanity
 
     # TODO: once `trio` get's an "obvious way" for req/resp we
     # should use it?
@@ -978,12 +1031,10 @@ async def process_messages(
                         cid=cid,
                         kwargs=kwargs,
                     ):
-                        kwargs |= {'req_chan': chan}
-
                         # XXX NOTE XXX don't start entire actor
                         # runtime cancellation if this actor is
                         # currently in debug mode!
-                        pdb_complete: trio.Event|None = _debug.DebugStatus.repl_release
+                        pdb_complete: trio.Event|None = debug.DebugStatus.repl_release
                         if pdb_complete:
                             await pdb_complete.wait()
 
@@ -998,14 +1049,14 @@ async def process_messages(
                                 cid,
                                 chan,
                                 actor.cancel,
-                                kwargs,
+                                kwargs | {'req_chan': chan},
                                 is_rpc=False,
                                 return_msg_type=CancelAck,
                             )
 
                         log.runtime(
-                            'Cancelling IPC transport msg-loop with peer:\n'
-                            f'|_{chan}\n'
+                            'Cancelling RPC-msg-loop with peer\n'
+                            f'->c}} {chan.aid.reprol()}@[{chan.maddr}]\n'
                         )
                         loop_cs.cancel()
                         break
@@ -1018,7 +1069,7 @@ async def process_messages(
                     ):
                         target_cid: str = kwargs['cid']
                         kwargs |= {
-                            'requesting_uid': chan.uid,
+                            'requesting_aid': chan.aid,
                             'ipc_msg': msg,
 
                             # XXX NOTE! ONLY the rpc-task-owning
@@ -1054,21 +1105,34 @@ async def process_messages(
                         ns=ns,
                         func=funcname,
                         kwargs=kwargs,  # type-spec this? see `msg.types`
-                        uid=actorid,
+                        uid=actor_uuid,
                     ):
+                        if actor_uuid != chan.aid.uid:
+                            raise RuntimeError(
+                                f'IPC <Start> msg <-> chan.aid mismatch!?\n'
+                                f'Channel.aid = {chan.aid!r}\n'
+                                f'Start.uid = {actor_uuid!r}\n'
+                            )
+                        # await debug.pause()
+                        op_repr: str = 'Start <=) '
+                        req_repr: str = _pformat.nest_from_op(
+                            input_op=op_repr,
+                            op_suffix='',
+                            nest_prefix='',
+                            text=f'{chan}',
+
+                            nest_indent=len(op_repr)-1,
+                            rm_from_first_ln='<',
+                            # ^XXX, subtract -1 to account for
+                            # <Channel
+                            # ^_chevron to be stripped
+                        )
                         start_status: str = (
-                            'Handling RPC `Start` request\n'
-                            f'<= peer: {actorid}\n\n'
-                            f'  |_{chan}\n'
-                            f'  |_cid: {cid}\n\n'
-                            # f'  |_{ns}.{funcname}({kwargs})\n'
-                            f'>> {actor.uid}\n'
-                            f'  |_{actor}\n'
-                            f'   -> nsp: `{ns}.{funcname}({kwargs})`\n'
-
-                            # f'  |_{ns}.{funcname}({kwargs})\n\n'
-
-                            # f'{pretty_struct.pformat(msg)}\n'
+                            'Handling RPC request\n'
+                            f'{req_repr}\n'
+                            f'\n'
+                            f'->{{ ipc-context-id: {cid!r}\n'
+                            f'->{{ nsp for fn: `{ns}.{funcname}({kwargs})`\n'
                         )
 
                         # runtime-internal endpoint: `Actor.<funcname>`
@@ -1097,10 +1161,6 @@ async def process_messages(
                                 await chan.send(err_msg)
                                 continue
 
-                        start_status += (
-                            f'   -> func: {func}\n'
-                        )
-
                         # schedule a task for the requested RPC function
                         # in the actor's main "service nursery".
                         #
@@ -1108,10 +1168,10 @@ async def process_messages(
                         # supervision isolation? would avoid having to
                         # manage RPC tasks individually in `._rpc_tasks`
                         # table?
-                        start_status += '   -> scheduling new task..\n'
+                        start_status += '->( scheduling new task..\n'
                         log.runtime(start_status)
                         try:
-                            ctx: Context = await actor._service_n.start(
+                            ctx: Context = await actor._service_tn.start(
                                 partial(
                                     _invoke,
                                     actor,
@@ -1192,12 +1252,24 @@ async def process_messages(
             # END-OF `async for`:
             # IPC disconnected via `trio.EndOfChannel`, likely
             # due to a (graceful) `Channel.aclose()`.
+
+            chan_op_repr: str = '<=x] '
+            chan_repr: str = _pformat.nest_from_op(
+                input_op=chan_op_repr,
+                op_suffix='',
+                nest_prefix='',
+                text=chan.pformat(),
+                nest_indent=len(chan_op_repr)-1,
+                rm_from_first_ln='<',
+            )
             log.runtime(
-                f'channel for {chan.uid} disconnected, cancelling RPC tasks\n'
-                f'|_{chan}\n'
+                f'IPC channel disconnected\n'
+                f'{chan_repr}\n'
+                f'\n'
+                f'->c) cancelling RPC tasks.\n'
             )
             await actor.cancel_rpc_tasks(
-                req_uid=actor.uid,
+                req_aid=actor.aid,
                 # a "self cancel" in terms of the lifetime of the
                 # IPC connection which is presumed to be the
                 # source of any requests for spawned tasks.
@@ -1239,7 +1311,7 @@ async def process_messages(
     ) as err:
 
         if nursery_cancelled_before_task:
-            sn: Nursery = actor._service_n
+            sn: Nursery = actor._service_tn
             assert sn and sn.cancel_scope.cancel_called  # sanity
             log.cancel(
                 f'Service nursery cancelled before it handled {funcname}'
@@ -1269,13 +1341,37 @@ async def process_messages(
     finally:
         # msg debugging for when he machinery is brokey
         if msg is None:
-            message: str = 'Exiting IPC msg loop without receiving a msg?'
+            message: str = 'Exiting RPC-loop without receiving a msg?'
         else:
+            task_op_repr: str = ')>'
+            task: trio.Task = trio.lowlevel.current_task()
+
+            # maybe add cancelled opt prefix
+            if task._cancel_status.effectively_cancelled:
+                task_op_repr = 'c' + task_op_repr
+
+            task_repr: str = _pformat.nest_from_op(
+                input_op=task_op_repr,
+                text=f'{task!r}',
+                nest_indent=1,
+            )
+            # chan_op_repr: str = '<=} '
+            # chan_repr: str = _pformat.nest_from_op(
+            #     input_op=chan_op_repr,
+            #     op_suffix='',
+            #     nest_prefix='',
+            #     text=chan.pformat(),
+            #     nest_indent=len(chan_op_repr)-1,
+            #     rm_from_first_ln='<',
+            # )
             message: str = (
-                'Exiting IPC msg loop with final msg\n\n'
-                f'<= peer: {chan.uid}\n'
-                f'  |_{chan}\n\n'
-                # f'{pretty_struct.pformat(msg)}'
+                f'Exiting RPC-loop with final msg\n'
+                f'\n'
+                # f'{chan_repr}\n'
+                f'{task_repr}\n'
+                f'\n'
+                f'{pretty_struct.pformat(msg)}'
+                f'\n'
             )
 
         log.runtime(message)

tractor/_runtime.py

@@ -35,6 +35,15 @@ for running all lower level spawning, supervision and msging layers:
   SC-transitive RPC via scheduling of `trio` tasks.
 - registration of newly spawned actors with the discovery sys.
 
+Glossary:
+--------
+ - tn: a `trio.Nursery` or "task nursery".
+ - an: an `ActorNursery` or "actor nursery".
+ - root: top/parent-most scope/task/process/actor (or other runtime
+         primitive) in a hierarchical tree.
+ - parent-ish: "higher-up" in the runtime-primitive hierarchy.
+ - child-ish: "lower-down" in the runtime-primitive hierarchy.
+
 '''
 from __future__ import annotations
 from contextlib import (
@@ -55,6 +64,7 @@ from typing import (
     TYPE_CHECKING,
 )
 import uuid
+import textwrap
 from types import ModuleType
 import warnings
 
@@ -73,6 +83,10 @@ from tractor.msg import (
     pretty_struct,
     types as msgtypes,
 )
+from .trionics import (
+    collapse_eg,
+    maybe_open_nursery,
+)
 from .ipc import (
     Channel,
     # IPCServer,  # causes cycles atm..
@@ -97,7 +111,10 @@ from ._exceptions import (
     MsgTypeError,
     unpack_error,
 )
-from .devx import _debug
+from .devx import (
+    debug,
+    pformat as _pformat
+)
 from ._discovery import get_registry
 from ._portal import Portal
 from . import _state
@@ -166,10 +183,11 @@ class Actor:
 
     msg_buffer_size: int = 2**6
 
-    # nursery placeholders filled in by `async_main()` after fork
-    _root_n: Nursery|None = None
-    _service_n: Nursery|None = None
-
+    # nursery placeholders filled in by `async_main()`,
+    # - after fork for subactors.
+    # - during boot for the root actor.
+    _root_tn: Nursery|None = None
+    _service_tn: Nursery|None = None
     _ipc_server: _server.IPCServer|None = None
 
     @property
@@ -206,7 +224,7 @@ class Actor:
         *,
         enable_modules: list[str] = [],
         loglevel: str|None = None,
-        registry_addrs: list[UnwrappedAddress]|None = None,
+        registry_addrs: list[Address]|None = None,
         spawn_method: str|None = None,
 
         # TODO: remove!
@@ -227,7 +245,7 @@ class Actor:
 
         # state
         self._cancel_complete = trio.Event()
-        self._cancel_called_by_remote: tuple[str, tuple]|None = None
+        self._cancel_called_by: tuple[str, tuple]|None = None
         self._cancel_called: bool = False
 
         # retreive and store parent `__main__` data which
@@ -237,7 +255,7 @@ class Actor:
         # TODO? only add this when `is_debug_mode() == True` no?
         # always include debugging tools module
         if _state.is_root_process():
-            enable_modules.append('tractor.devx._debug')
+            enable_modules.append('tractor.devx.debug._tty_lock')
 
         self.enable_modules: dict[str, str] = get_mod_nsps2fps(
             mod_ns_paths=enable_modules,
@@ -249,11 +267,12 @@ class Actor:
         if arbiter_addr is not None:
             warnings.warn(
                 '`Actor(arbiter_addr=<blah>)` is now deprecated.\n'
-                'Use `registry_addrs: list[tuple]` instead.',
+                'Use `registry_addrs: list[Address]` instead.',
                 DeprecationWarning,
                 stacklevel=2,
             )
-            registry_addrs: list[UnwrappedAddress] = [arbiter_addr]
+
+            registry_addrs: list[Address] = [wrap_address(arbiter_addr)]
 
         # marked by the process spawning backend at startup
         # will be None for the parent most process started manually
@@ -292,8 +311,10 @@ class Actor:
         # input via the validator.
         self._reg_addrs: list[UnwrappedAddress] = []
         if registry_addrs:
-            self.reg_addrs: list[UnwrappedAddress] = registry_addrs
-            _state._runtime_vars['_registry_addrs'] = registry_addrs
+            _state._runtime_vars['_registry_addrs'] = self.reg_addrs = [
+                addr.unwrap()
+                for addr in registry_addrs
+            ]
 
     @property
     def aid(self) -> msgtypes.Aid:
@@ -339,46 +360,125 @@ class Actor:
     def pid(self) -> int:
         return self._aid.pid
 
-    def pformat(self) -> str:
-        ds: str = '='
-        parent_uid: tuple|None = None
-        if rent_chan := self._parent_chan:
-            parent_uid = rent_chan.uid
+    @property
+    def repr_state(self) -> str:
+        if self.cancel_complete:
+            return 'cancelled'
+
+        elif canceller := self.cancel_caller:
+                return f' and cancel-called by {canceller}'
+
+        else:
+            return 'running'
+
+    def pformat(
+        self,
+        ds: str = ': ',
+        indent: int = 0,
+        privates: bool = False,
+    ) -> str:
+
+        fmtstr: str = f'|_id: {self.aid.reprol()!r}\n'
+        if privates:
+            aid_nest_prefix: str = '|_aid='
+            aid_field_repr: str = _pformat.nest_from_op(
+                input_op='',
+                text=pretty_struct.pformat(
+                    struct=self.aid,
+                    field_indent=2,
+                ),
+                op_suffix='',
+                nest_prefix=aid_nest_prefix,
+                nest_indent=0,
+            )
+            fmtstr: str = f'{aid_field_repr}'
+
+        if rent_chan := self._parent_chan:
+            fmtstr += (
+                f"|_parent{ds}{rent_chan.aid.reprol()}\n"
+            )
 
-        peers: list = []
         server: _server.IPCServer = self.ipc_server
         if server:
-            peers: list[tuple] = list(server._peer_connected)
+            if privates:
+                server_repr: str = self._ipc_server.pformat(
+                    privates=privates,
+                )
+                # create field ln as a key-header indented under
+                # and up to the section's key prefix.
+                # ^XXX if we were to indent `repr(Server)` to
+                # '<key>: '
+                #  _here_^
+                server_repr: str = _pformat.nest_from_op(
+                    input_op='',  # nest as sub-obj
+                    op_suffix='',
+                    text=server_repr,
+                )
+                fmtstr += (
+                    f"{server_repr}"
+                )
+            else:
+                fmtstr += (
+                    f'|_ipc: {server.repr_state!r}\n'
+                )
 
-        fmtstr: str = (
-            f' |_id: {self.aid!r}\n'
-            # f"   aid{ds}{self.aid!r}\n"
-            f"   parent{ds}{parent_uid}\n"
-            f'\n'
-            f' |_ipc: {len(peers)!r} connected peers\n'
-            f"   peers{ds}{peers!r}\n"
-            f"   ipc_server{ds}{self._ipc_server}\n"
-            f'\n'
-            f' |_rpc: {len(self._rpc_tasks)} tasks\n'
-            f"   ctxs{ds}{len(self._contexts)}\n"
-            f'\n'
-            f' |_runtime: ._task{ds}{self._task!r}\n'
-            f'   _spawn_method{ds}{self._spawn_method}\n'
-            f'   _actoruid2nursery{ds}{self._actoruid2nursery}\n'
-            f'   _forkserver_info{ds}{self._forkserver_info}\n'
-            f'\n'
-            f' |_state: "TODO: .repr_state()"\n'
-            f'   _cancel_complete{ds}{self._cancel_complete}\n'
-            f'   _cancel_called_by_remote{ds}{self._cancel_called_by_remote}\n'
-            f'   _cancel_called{ds}{self._cancel_called}\n'
+        fmtstr += (
+            f'|_rpc: {len(self._rpc_tasks)} active tasks\n'
         )
-        return (
-            '<Actor(\n'
-            +
-            fmtstr
-            +
-            ')>\n'
+
+        # TODO, actually fix the .repr_state impl/output?
+        # append ipc-ctx state summary
+        # ctxs: dict = self._contexts
+        # if ctxs:
+        #     ctx_states: dict[str, int] = {}
+        #     for ctx in self._contexts.values():
+        #         ctx_state: str = ctx.repr_state
+        #         cnt = ctx_states.setdefault(ctx_state, 0)
+        #         ctx_states[ctx_state] = cnt + 1
+
+        #     fmtstr += (
+        #         f"  ctxs{ds}{ctx_states}\n"
+        #     )
+
+        # runtime-state
+        task_name: str = '<dne>'
+        if task := self._task:
+            task_name: str = task.name
+        fmtstr += (
+            # TODO, this just like ctx?
+            f'|_state: {self.repr_state!r}\n'
+            f'  task: {task_name}\n'
+            f'  loglevel: {self.loglevel!r}\n'
+            f'  subactors_spawned: {len(self._actoruid2nursery)}\n'
         )
+        if not _state.is_root_process():
+            fmtstr += f'  spawn_method: {self._spawn_method!r}\n'
+
+        if privates:
+            fmtstr += (
+                # f'  actoruid2nursery{ds}{self._actoruid2nursery}\n'
+                f'  cancel_complete{ds}{self._cancel_complete}\n'
+                f'  cancel_called_by_remote{ds}{self._cancel_called_by}\n'
+                f'  cancel_called{ds}{self._cancel_called}\n'
+            )
+
+        if fmtstr:
+            fmtstr: str = textwrap.indent(
+                text=fmtstr,
+                prefix=' '*(1 + indent),
+            )
+
+        _repr: str = (
+            f'<{type(self).__name__}(\n'
+            f'{fmtstr}'
+            f')>\n'
+        )
+        if indent:
+            _repr: str = textwrap.indent(
+                text=_repr,
+                prefix=' '*indent,
+            )
+        return _repr
 
     __repr__ = pformat
 
@@ -386,7 +486,11 @@ class Actor:
     def reg_addrs(self) -> list[UnwrappedAddress]:
         '''
         List of (socket) addresses for all known (and contactable)
-        registry actors.
+        registry-service actors in "unwrapped" (i.e. IPC interchange
+        wire-compat) form.
+
+        If you are looking for the "wrapped" address form, use
+        `.registry_addrs` instead.
 
         '''
         return self._reg_addrs
@@ -405,8 +509,14 @@ class Actor:
 
         self._reg_addrs = addrs
 
+    @property
+    def registry_addrs(self) -> list[Address]:
+        return [wrap_address(uw_addr)
+                for uw_addr in self.reg_addrs]
+
     def load_modules(
         self,
+
     ) -> None:
         '''
         Load explicitly enabled python modules from local fs after
@@ -453,6 +563,14 @@ class Actor:
             )
             raise
 
+    # ?TODO, factor this meth-iface into a new `.rpc` subsys primitive?
+    # - _get_rpc_func(),
+    # - _deliver_ctx_payload(),
+    # - get_context(),
+    # - start_remote_task(),
+    # - cancel_rpc_tasks(),
+    # - _cancel_task(),
+    #
     def _get_rpc_func(self, ns, funcname):
         '''
         Try to lookup and return a target RPC func from the
@@ -496,11 +614,11 @@ class Actor:
         queue.
 
         '''
-        uid: tuple[str, str] = chan.uid
-        assert uid, f"`chan.uid` can't be {uid}"
+        aid: msgtypes.Aid = chan.aid
+        assert aid, f"`chan.aid` can't be {aid}"
         try:
             ctx: Context = self._contexts[(
-                uid,
+                aid.uid,
                 cid,
 
                 # TODO: how to determine this tho?
@@ -511,7 +629,7 @@ class Actor:
                 'Ignoring invalid IPC msg!?\n'
                 f'Ctx seems to not/no-longer exist??\n'
                 f'\n'
-                f'<=? {uid}\n'
+                f'<=? {aid.reprol()!r}\n'
                 f'  |_{pretty_struct.pformat(msg)}\n'
             )
             match msg:
@@ -560,6 +678,7 @@ class Actor:
           msging session's lifetime.
 
         '''
+        # ?TODO, use Aid here as well?
         actor_uid = chan.uid
         assert actor_uid
         try:
@@ -902,12 +1021,64 @@ class Actor:
         the RPC service nursery.
 
         '''
-        assert self._service_n
-        self._service_n.start_soon(
+        actor_repr: str = _pformat.nest_from_op(
+            input_op='>c(',
+            text=self.pformat(),
+            nest_indent=1,
+        )
+        log.cancel(
+            'Actor.cancel_soon()` was called!\n'
+            f'>> scheduling `Actor.cancel()`\n'
+            f'{actor_repr}'
+        )
+        assert self._service_tn
+        self._service_tn.start_soon(
             self.cancel,
             None,  # self cancel all rpc tasks
         )
 
+        # schedule a "canceller task" in the `._root_tn` once the
+        # `._service_tn` is fully shutdown; task waits for child-ish
+        # scopes to fully exit then finally cancels its parent,
+        # root-most, scope.
+        async def cancel_root_tn_after_services():
+            log.runtime(
+                'Waiting on service-tn to cancel..\n'
+                f'c>)\n'
+                f'|_{self._service_tn.cancel_scope!r}\n'
+            )
+            await self._cancel_complete.wait()
+            log.cancel(
+                f'`._service_tn` cancelled\n'
+                f'>c)\n'
+                f'|_{self._service_tn.cancel_scope!r}\n'
+                f'\n'
+                f'>> cancelling `._root_tn`\n'
+                f'c>(\n'
+                f' |_{self._root_tn.cancel_scope!r}\n'
+            )
+            self._root_tn.cancel_scope.cancel()
+
+        self._root_tn.start_soon(
+            cancel_root_tn_after_services
+        )
+
+    @property
+    def cancel_complete(self) -> bool:
+        return self._cancel_complete.is_set()
+
+    @property
+    def cancel_called(self) -> bool:
+        '''
+        Was this actor requested to cancel by a remote peer actor.
+
+        '''
+        return self._cancel_called_by is not None
+
+    @property
+    def cancel_caller(self) -> msgtypes.Aid|None:
+        return self._cancel_called_by
+
     async def cancel(
         self,
 
@@ -932,20 +1103,18 @@ class Actor:
 
         '''
         (
-            requesting_uid,
-            requester_type,
+            requesting_aid,  # Aid
+            requester_type,  # str
             req_chan,
             log_meth,
         ) = (
-            req_chan.uid,
+            req_chan.aid,
             'peer',
             req_chan,
             log.cancel,
-
         ) if req_chan else (
-
             # a self cancel of ALL rpc tasks
-            self.uid,
+            self.aid,
             'self',
             self,
             log.runtime,
@@ -953,14 +1122,14 @@ class Actor:
         # TODO: just use the new `Context.repr_rpc: str` (and
         # other) repr fields instead of doing this all manual..
         msg: str = (
-            f'Actor-runtime cancel request from {requester_type}\n\n'
-            f'<=c) {requesting_uid}\n'
-            f'  |_{self}\n'
+            f'Actor-runtime cancel request from {requester_type!r}\n'
             f'\n'
+            f'<=c)\n'
+            f'{self}'
         )
 
         # TODO: what happens here when we self-cancel tho?
-        self._cancel_called_by_remote: tuple = requesting_uid
+        self._cancel_called_by: tuple = requesting_aid
         self._cancel_called = True
 
         # cancel all ongoing rpc tasks
@@ -968,7 +1137,7 @@ class Actor:
 
             # kill any debugger request task to avoid deadlock
             # with the root actor in this tree
-            debug_req = _debug.DebugStatus
+            debug_req = debug.DebugStatus
             lock_req_ctx: Context = debug_req.req_ctx
             if (
                 lock_req_ctx
@@ -978,7 +1147,7 @@ class Actor:
                 msg += (
                     f'\n'
                     f'-> Cancelling active debugger request..\n'
-                    f'|_{_debug.Lock.repr()}\n\n'
+                    f'|_{debug.Lock.repr()}\n\n'
                     f'|_{lock_req_ctx}\n\n'
                 )
                 # lock_req_ctx._scope.cancel()
@@ -988,7 +1157,7 @@ class Actor:
 
             # self-cancel **all** ongoing RPC tasks
             await self.cancel_rpc_tasks(
-                req_uid=requesting_uid,
+                req_aid=requesting_aid,
                 parent_chan=None,
             )
 
@@ -998,26 +1167,18 @@ class Actor:
                 await ipc_server.wait_for_shutdown()
 
             # cancel all rpc tasks permanently
-            if self._service_n:
-                self._service_n.cancel_scope.cancel()
+            if self._service_tn:
+                self._service_tn.cancel_scope.cancel()
 
         log_meth(msg)
         self._cancel_complete.set()
         return True
 
-    # XXX: hard kill logic if needed?
-    # def _hard_mofo_kill(self):
-    #     # If we're the root actor or zombied kill everything
-    #     if self._parent_chan is None:  # TODO: more robust check
-    #         root = trio.lowlevel.current_root_task()
-    #         for n in root.child_nurseries:
-    #             n.cancel_scope.cancel()
-
     async def _cancel_task(
         self,
         cid: str,
         parent_chan: Channel,
-        requesting_uid: tuple[str, str]|None,
+        requesting_aid: msgtypes.Aid|None,
 
         ipc_msg: dict|None|bool = False,
 
@@ -1055,7 +1216,7 @@ class Actor:
             log.runtime(
                 'Cancel request for invalid RPC task.\n'
                 'The task likely already completed or was never started!\n\n'
-                f'<= canceller: {requesting_uid}\n'
+                f'<= canceller: {requesting_aid}\n'
                 f'=> {cid}@{parent_chan.uid}\n'
                 f'  |_{parent_chan}\n'
             )
@@ -1063,9 +1224,12 @@ class Actor:
 
         log.cancel(
             'Rxed cancel request for RPC task\n'
-            f'<=c) {requesting_uid}\n'
-            f' |_{ctx._task}\n'
-            f'    >> {ctx.repr_rpc}\n'
+            f'{ctx._task!r} <=c) {requesting_aid}\n'
+            f'|_>> {ctx.repr_rpc}\n'
+
+            # f'|_{ctx._task}\n'
+            # f'   >> {ctx.repr_rpc}\n'
+
             # f'=> {ctx._task}\n'
             # f'  >> Actor._cancel_task() => {ctx._task}\n'
             # f'  |_ {ctx._task}\n\n'
@@ -1086,9 +1250,9 @@ class Actor:
         )
         if (
             ctx._canceller is None
-            and requesting_uid
+            and requesting_aid
         ):
-            ctx._canceller: tuple = requesting_uid
+            ctx._canceller: tuple = requesting_aid.uid
 
         # TODO: pack the RPC `{'cmd': <blah>}` msg into a ctxc and
         # then raise and pack it here?
@@ -1114,7 +1278,7 @@ class Actor:
 
         # wait for _invoke to mark the task complete
         flow_info: str = (
-            f'<= canceller: {requesting_uid}\n'
+            f'<= canceller: {requesting_aid}\n'
             f'=> ipc-parent: {parent_chan}\n'
             f'|_{ctx}\n'
         )
@@ -1131,7 +1295,7 @@ class Actor:
 
     async def cancel_rpc_tasks(
         self,
-        req_uid: tuple[str, str],
+        req_aid: msgtypes.Aid,
 
         # NOTE: when None is passed we cancel **all** rpc
         # tasks running in this actor!
@@ -1141,14 +1305,14 @@ class Actor:
         '''
         Cancel all ongoing RPC tasks owned/spawned for a given
         `parent_chan: Channel` or simply all tasks (inside
-        `._service_n`) when `parent_chan=None`.
+        `._service_tn`) when `parent_chan=None`.
 
         '''
         tasks: dict = self._rpc_tasks
         if not tasks:
             log.runtime(
                 'Actor has no cancellable RPC tasks?\n'
-                f'<= canceller: {req_uid}\n'
+                f'<= canceller: {req_aid.reprol()}\n'
             )
             return
 
@@ -1188,7 +1352,7 @@ class Actor:
         )
         log.cancel(
             f'Cancelling {descr} RPC tasks\n\n'
-            f'<=c) {req_uid} [canceller]\n'
+            f'<=c) {req_aid} [canceller]\n'
             f'{rent_chan_repr}'
             f'c)=> {self.uid} [cancellee]\n'
             f'  |_{self} [with {len(tasks)} tasks]\n'
@@ -1216,7 +1380,7 @@ class Actor:
             await self._cancel_task(
                 cid,
                 task_caller_chan,
-                requesting_uid=req_uid,
+                requesting_aid=req_aid,
             )
 
         if tasks:
@@ -1244,25 +1408,13 @@ class Actor:
         '''
         return self.accept_addrs[0]
 
-    def get_parent(self) -> Portal:
-        '''
-        Return a `Portal` to our parent.
-
-        '''
-        assert self._parent_chan, "No parent channel for this actor?"
-        return Portal(self._parent_chan)
-
-    def get_chans(
-        self,
-        uid: tuple[str, str],
-
-    ) -> list[Channel]:
-        '''
-        Return all IPC channels to the actor with provided `uid`.
-
-        '''
-        return self._peers[uid]
-
+    # TODO, this should delegate ONLY to the
+    # `._spawn_spec._runtime_vars: dict` / `._state` APIs?
+    #
+    # XXX, AH RIGHT that's why..
+    #   it's bc we pass this as a CLI flag to the child.py precisely
+    #   bc we need the bootstrapping pre `async_main()`.. but maybe
+    #   keep this as an impl deat and not part of the pub iface impl?
     def is_infected_aio(self) -> bool:
         '''
         If `True`, this actor is running `trio` in guest mode on
@@ -1273,6 +1425,23 @@ class Actor:
         '''
         return self._infected_aio
 
+    # ?TODO, is this the right type for this method?
+    def get_parent(self) -> Portal:
+        '''
+        Return a `Portal` to our parent.
+
+        '''
+        assert self._parent_chan, "No parent channel for this actor?"
+        return Portal(self._parent_chan)
+
+    # XXX: hard kill logic if needed?
+    # def _hard_mofo_kill(self):
+    #     # If we're the root actor or zombied kill everything
+    #     if self._parent_chan is None:  # TODO: more robust check
+    #         root = trio.lowlevel.current_root_task()
+    #         for n in root.child_nurseries:
+    #             n.cancel_scope.cancel()
+
 
 async def async_main(
     actor: Actor,
@@ -1308,7 +1477,7 @@ async def async_main(
 
     # attempt to retreive ``trio``'s sigint handler and stash it
     # on our debugger state.
-    _debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
+    debug.DebugStatus._trio_handler = signal.getsignal(signal.SIGINT)
 
     is_registered: bool = False
     try:
@@ -1316,6 +1485,8 @@ async def async_main(
         # establish primary connection with immediate parent
         actor._parent_chan: Channel|None = None
 
+        # is this a sub-actor?
+        # get runtime info from parent.
         if parent_addr is not None:
             (
                 actor._parent_chan,
@@ -1346,46 +1517,55 @@ async def async_main(
                     accept_addrs.append(addr.unwrap())
 
         assert accept_addrs
-        # The "root" nursery ensures the channel with the immediate
-        # parent is kept alive as a resilient service until
-        # cancellation steps have (mostly) occurred in
-        # a deterministic way.
-        async with trio.open_nursery(
-            strict_exception_groups=False,
-        ) as root_nursery:
-            actor._root_n = root_nursery
-            assert actor._root_n
+
+        ya_root_tn: bool = bool(actor._root_tn)
+        ya_service_tn: bool = bool(actor._service_tn)
+
+        # NOTE, a top-most "root" nursery in each actor-process
+        # enables a lifetime priority for the IPC-channel connection
+        # with a sub-actor's immediate parent. I.e. this connection
+        # is kept alive as a resilient service connection until all
+        # other machinery has exited, cancellation of all
+        # embedded/child scopes have completed. This helps ensure
+        # a deterministic (and thus "graceful")
+        # first-class-supervision style teardown where a parent actor
+        # (vs. say peers) is always the last to be contacted before
+        # disconnect.
+        root_tn: trio.Nursery
+        async with (
+            collapse_eg(),
+            maybe_open_nursery(
+                nursery=actor._root_tn,
+            ) as root_tn,
+        ):
+            if ya_root_tn:
+                assert root_tn is actor._root_tn
+            else:
+                actor._root_tn = root_tn
 
             ipc_server: _server.IPCServer
             async with (
-                trio.open_nursery(
-                    strict_exception_groups=False,
-                ) as service_nursery,
-
+                collapse_eg(),
+                maybe_open_nursery(
+                    nursery=actor._service_tn,
+                ) as service_tn,
                 _server.open_ipc_server(
-                    parent_tn=service_nursery,
-                    stream_handler_tn=service_nursery,
+                    parent_tn=service_tn,  # ?TODO, why can't this be the root-tn
+                    stream_handler_tn=service_tn,
                 ) as ipc_server,
-                # ) as actor._ipc_server,
-                # ^TODO? prettier?
 
             ):
-                # This nursery is used to handle all inbound
-                # connections to us such that if the TCP server
-                # is killed, connections can continue to process
-                # in the background until this nursery is cancelled.
-                actor._service_n = service_nursery
+                if ya_service_tn:
+                    assert service_tn is actor._service_tn
+                else:
+                    # This nursery is used to handle all inbound
+                    # connections to us such that if the TCP server
+                    # is killed, connections can continue to process
+                    # in the background until this nursery is cancelled.
+                    actor._service_tn = service_tn
+
+                # set after allocate
                 actor._ipc_server = ipc_server
-                assert (
-                    actor._service_n
-                    and (
-                        actor._service_n
-                        is
-                        actor._ipc_server._parent_tn
-                        is
-                        ipc_server._stream_handler_tn
-                    )
-                )
 
                 # load exposed/allowed RPC modules
                 # XXX: do this **after** establishing a channel to the parent
@@ -1401,7 +1581,7 @@ async def async_main(
                 # try:
                 #     actor.load_modules()
                 # except ModuleNotFoundError as err:
-                #     _debug.pause_from_sync()
+                #     debug.pause_from_sync()
                 #     import pdbp; pdbp.set_trace()
                 #     raise
 
@@ -1411,13 +1591,11 @@ async def async_main(
                 # - root actor: the ``accept_addr`` passed to this method
 
                 # TODO: why is this not with the root nursery?
+                # - see above that the `._service_tn` is what's used?
                 try:
-                    log.runtime(
-                        'Booting IPC server'
-                    )
                     eps: list = await ipc_server.listen_on(
                         accept_addrs=accept_addrs,
-                        stream_handler_nursery=service_nursery,
+                        stream_handler_nursery=service_tn,
                     )
                     log.runtime(
                         f'Booted IPC server\n'
@@ -1425,7 +1603,7 @@ async def async_main(
                     )
                     assert (
                         (eps[0].listen_tn)
-                        is not service_nursery
+                        is not service_tn
                     )
 
                 except OSError as oserr:
@@ -1433,7 +1611,7 @@ async def async_main(
                     # tranport address bind errors - normally it's
                     # something silly like the wrong socket-address
                     # passed via a config or CLI Bo
-                    entered_debug: bool = await _debug._maybe_enter_pm(
+                    entered_debug: bool = await debug._maybe_enter_pm(
                         oserr,
                     )
                     if not entered_debug:
@@ -1446,18 +1624,6 @@ async def async_main(
                 # TODO, just read direct from ipc_server?
                 accept_addrs: list[UnwrappedAddress] = actor.accept_addrs
 
-                # NOTE: only set the loopback addr for the 
-                # process-tree-global "root" mailbox since
-                # all sub-actors should be able to speak to
-                # their root actor over that channel.
-                if _state._runtime_vars['_is_root']:
-                    raddrs: list[Address] = _state._runtime_vars['_root_addrs']
-                    for addr in accept_addrs:
-                        waddr: Address = wrap_address(addr)
-                        raddrs.append(addr)
-                    else:
-                        _state._runtime_vars['_root_mailbox'] = raddrs[0]
-
                 # Register with the arbiter if we're told its addr
                 log.runtime(
                     f'Registering `{actor.name}` => {pformat(accept_addrs)}\n'
@@ -1473,8 +1639,9 @@ async def async_main(
                         waddr = wrap_address(addr)
                         assert waddr.is_valid
                     except AssertionError:
-                        await _debug.pause()
+                        await debug.pause()
 
+                    # !TODO, get rid of the local-portal crap XD
                     async with get_registry(addr) as reg_portal:
                         for accept_addr in accept_addrs:
                             accept_addr = wrap_address(accept_addr)
@@ -1499,7 +1666,7 @@ async def async_main(
                 # start processing parent requests until our channel
                 # server is 100% up and running.
                 if actor._parent_chan:
-                    await root_nursery.start(
+                    await root_tn.start(
                         partial(
                             _rpc.process_messages,
                             chan=actor._parent_chan,
@@ -1511,8 +1678,9 @@ async def async_main(
                     # 'Blocking on service nursery to exit..\n'
                 )
             log.runtime(
-                "Service nursery complete\n"
-                "Waiting on root nursery to complete"
+                'Service nursery complete\n'
+                '\n'
+                '->} waiting on root nursery to complete..\n'
             )
 
         # Blocks here as expected until the root nursery is
@@ -1567,6 +1735,7 @@ async def async_main(
     finally:
         teardown_report: str = (
             'Main actor-runtime task completed\n'
+            '\n'
         )
 
         # ?TODO? should this be in `._entry`/`._root` mods instead?
@@ -1590,25 +1759,26 @@ async def async_main(
             # prevents any `infected_aio` actor from continuing
             # and any callbacks in the `ls` here WILL NOT be
             # called!!
-            # await _debug.pause(shield=True)
+            # await debug.pause(shield=True)
 
         ls.close()
 
         # XXX TODO but hard XXX
         # we can't actually do this bc the debugger uses the
-        # _service_n to spawn the lock task, BUT, in theory if we had
+        # _service_tn to spawn the lock task, BUT, in theory if we had
         # the root nursery surround this finally block it might be
         # actually possible to debug THIS machinery in the same way
         # as user task code?
         #
         # if actor.name == 'brokerd.ib':
         #     with CancelScope(shield=True):
-        #         await _debug.breakpoint()
+        #         await debug.breakpoint()
 
         # Unregister actor from the registry-sys / registrar.
         if (
             is_registered
-            and not actor.is_registrar
+            and
+            not actor.is_registrar
         ):
             failed: bool = False
             for addr in actor.reg_addrs:
@@ -1643,23 +1813,30 @@ async def async_main(
             ipc_server.has_peers(check_chans=True)
         ):
             teardown_report += (
-                f'-> Waiting for remaining peers {ipc_server._peers} to clear..\n'
+                f'-> Waiting for remaining peers to clear..\n'
+                f'   {pformat(ipc_server._peers)}'
             )
             log.runtime(teardown_report)
-            await ipc_server.wait_for_no_more_peers(
-                shield=True,
-            )
+            await ipc_server.wait_for_no_more_peers()
 
         teardown_report += (
-            '-> All peer channels are complete\n'
+            '-]> all peer channels are complete.\n'
         )
 
+    # op_nested_actor_repr: str = _pformat.nest_from_op(
+    #     input_op=')>',
+    #     text=actor.pformat(),
+    #     nest_prefix='|_',
+    #     nest_indent=1,  # under >
+    # )
     teardown_report += (
-        'Actor runtime exiting\n'
-        f'>)\n'
-        f'|_{actor}\n'
+        '-)> actor runtime main task exit.\n'
+        # f'{op_nested_actor_repr}'
     )
-    log.info(teardown_report)
+    # if _state._runtime_vars['_is_root']:
+    #     log.info(teardown_report)
+    # else:
+    log.runtime(teardown_report)
 
 
 # TODO: rename to `Registry` and move to `.discovery._registry`!
@@ -1792,7 +1969,7 @@ class Arbiter(Actor):
         waddr: Address = wrap_address(addr)
         if not waddr.is_valid:
             # should never be 0-dynamic-os-alloc
-            await _debug.pause()
+            await debug.pause()
 
         self._registry[uid] = addr
 

tractor/_spawn.py

@@ -34,9 +34,9 @@ from typing import (
 import trio
 from trio import TaskStatus
 
-from .devx._debug import (
-    maybe_wait_for_debugger,
-    acquire_debug_lock,
+from .devx import (
+    debug,
+    pformat as _pformat
 )
 from tractor._state import (
     current_actor,
@@ -51,14 +51,17 @@ from tractor._portal import Portal
 from tractor._runtime import Actor
 from tractor._entry import _mp_main
 from tractor._exceptions import ActorFailure
-from tractor.msg.types import (
-    Aid,
-    SpawnSpec,
+from tractor.msg import (
+    types as msgtypes,
+    pretty_struct,
 )
 
 
 if TYPE_CHECKING:
-    from ipc import IPCServer
+    from ipc import (
+        _server,
+        Channel,
+    )
     from ._supervise import ActorNursery
     ProcessType = TypeVar('ProcessType', mp.Process, trio.Process)
 
@@ -233,10 +236,6 @@ async def hard_kill(
     # whilst also hacking on it XD
     # terminate_after: int = 99999,
 
-    # NOTE: for mucking with `.pause()`-ing inside the runtime
-    # whilst also hacking on it XD
-    # terminate_after: int = 99999,
-
 ) -> None:
     '''
     Un-gracefully terminate an OS level `trio.Process` after timeout.
@@ -298,6 +297,23 @@ async def hard_kill(
     # zombies (as a feature) we ask the OS to do send in the
     # removal swad as the last resort.
     if cs.cancelled_caught:
+
+        # TODO? attempt at intermediary-rent-sub
+        # with child in debug lock?
+        # |_https://github.com/goodboy/tractor/issues/320
+        #
+        # if not is_root_process():
+        #     log.warning(
+        #         'Attempting to acquire debug-REPL-lock before zombie reap!'
+        #     )
+        #     with trio.CancelScope(shield=True):
+        #         async with debug.acquire_debug_lock(
+        #             subactor_uid=current_actor().uid,
+        #         ) as _ctx:
+        #             log.warning(
+        #                 'Acquired debug lock, child ready to be killed ??\n'
+        #             )
+
         # TODO: toss in the skynet-logo face as ascii art?
         log.critical(
             # 'Well, the #ZOMBIE_LORD_IS_HERE# to collect\n'
@@ -328,20 +344,21 @@ async def soft_kill(
     see `.hard_kill()`).
 
     '''
-    peer_aid: Aid = portal.channel.aid
+    chan: Channel = portal.channel
+    peer_aid: msgtypes.Aid = chan.aid
     try:
         log.cancel(
             f'Soft killing sub-actor via portal request\n'
             f'\n'
-            f'(c=> {peer_aid}\n'
-            f'  |_{proc}\n'
+            f'c)=> {peer_aid.reprol()}@[{chan.maddr}]\n'
+            f'   |_{proc}\n'
         )
         # wait on sub-proc to signal termination
         await wait_func(proc)
 
     except trio.Cancelled:
         with trio.CancelScope(shield=True):
-            await maybe_wait_for_debugger(
+            await debug.maybe_wait_for_debugger(
                 child_in_debug=_runtime_vars.get(
                     '_debug_mode', False
                 ),
@@ -465,7 +482,7 @@ async def trio_proc(
         "--uid",
         # TODO, how to pass this over "wire" encodings like
         # cmdline args?
-        # -[ ] maybe we can add an `Aid.min_tuple()` ?
+        # -[ ] maybe we can add an `msgtypes.Aid.min_tuple()` ?
         str(subactor.uid),
         # Address the child must connect to on startup
         "--parent_addr",
@@ -483,13 +500,14 @@ async def trio_proc(
 
     cancelled_during_spawn: bool = False
     proc: trio.Process|None = None
-    ipc_server: IPCServer = actor_nursery._actor.ipc_server
+    ipc_server: _server.Server = actor_nursery._actor.ipc_server
     try:
         try:
             proc: trio.Process = await trio.lowlevel.open_process(spawn_cmd, **proc_kwargs)
             log.runtime(
-                'Started new child\n'
-                f'|_{proc}\n'
+                f'Started new child subproc\n'
+                f'(>\n'
+                f' |_{proc}\n'
             )
 
             # wait for actor to spawn and connect back to us
@@ -507,10 +525,10 @@ async def trio_proc(
                 with trio.CancelScope(shield=True):
                     # don't clobber an ongoing pdb
                     if is_root_process():
-                        await maybe_wait_for_debugger()
+                        await debug.maybe_wait_for_debugger()
 
                     elif proc is not None:
-                        async with acquire_debug_lock(subactor.uid):
+                        async with debug.acquire_debug_lock(subactor.uid):
                             # soft wait on the proc to terminate
                             with trio.move_on_after(0.5):
                                 await proc.wait()
@@ -528,14 +546,19 @@ async def trio_proc(
 
         # send a "spawning specification" which configures the
         # initial runtime state of the child.
-        sspec = SpawnSpec(
+        sspec = msgtypes.SpawnSpec(
             _parent_main_data=subactor._parent_main_data,
             enable_modules=subactor.enable_modules,
             reg_addrs=subactor.reg_addrs,
             bind_addrs=bind_addrs,
             _runtime_vars=_runtime_vars,
         )
-        log.runtime(f'Sending spawn spec: {str(sspec)}')
+        log.runtime(
+            f'Sending spawn spec to child\n'
+            f'{{}}=> {chan.aid.reprol()!r}\n'
+            f'\n'
+            f'{pretty_struct.pformat(sspec)}\n'
+        )
         await chan.send(sspec)
 
         # track subactor in current nursery
@@ -563,7 +586,7 @@ async def trio_proc(
             # condition.
             await soft_kill(
                 proc,
-                trio.Process.wait,
+                trio.Process.wait,  # XXX, uses `pidfd_open()` below.
                 portal
             )
 
@@ -571,8 +594,7 @@ async def trio_proc(
             # tandem if not done already
             log.cancel(
                 'Cancelling portal result reaper task\n'
-                f'>c)\n'
-                f' |_{subactor.uid}\n'
+                f'c)> {subactor.aid.reprol()!r}\n'
             )
             nursery.cancel_scope.cancel()
 
@@ -581,21 +603,24 @@ async def trio_proc(
         # allowed! Do this **after** cancellation/teardown to avoid
         # killing the process too early.
         if proc:
+            reap_repr: str = _pformat.nest_from_op(
+                input_op='>x)',
+                text=subactor.pformat(),
+            )
             log.cancel(
                 f'Hard reap sequence starting for subactor\n'
-                f'>x)\n'
-                f' |_{subactor}@{subactor.uid}\n'
+                f'{reap_repr}'
             )
 
             with trio.CancelScope(shield=True):
                 # don't clobber an ongoing pdb
                 if cancelled_during_spawn:
                     # Try again to avoid TTY clobbering.
-                    async with acquire_debug_lock(subactor.uid):
+                    async with debug.acquire_debug_lock(subactor.uid):
                         with trio.move_on_after(0.5):
                             await proc.wait()
 
-                await maybe_wait_for_debugger(
+                await debug.maybe_wait_for_debugger(
                     child_in_debug=_runtime_vars.get(
                         '_debug_mode', False
                     ),
@@ -624,7 +649,7 @@ async def trio_proc(
                 #     acquire the lock and get notified of who has it,
                 #     check that uid against our known children?
                 # this_uid: tuple[str, str] = current_actor().uid
-                # await acquire_debug_lock(this_uid)
+                # await debug.acquire_debug_lock(this_uid)
 
                 if proc.poll() is None:
                     log.cancel(f"Attempting to hard kill {proc}")
@@ -727,7 +752,7 @@ async def mp_proc(
 
     log.runtime(f"Started {proc}")
 
-    ipc_server: IPCServer = actor_nursery._actor.ipc_server
+    ipc_server: _server.Server = actor_nursery._actor.ipc_server
     try:
         # wait for actor to spawn and connect back to us
         # channel should have handshake completed by the

tractor/_state.py

@@ -48,8 +48,9 @@ _current_actor: Actor|None = None  # type: ignore # noqa
 _last_actor_terminated: Actor|None = None
 
 # TODO: mk this a `msgspec.Struct`!
+# -[ ] type out all fields obvi!
+# -[ ] (eventually) mk wire-ready for monitoring?
 _runtime_vars: dict[str, Any] = {
-    '_debug_mode': False,
     # root of actor-process tree info
     '_is_root': False,  # bool
     '_root_mailbox': (None, None),  # tuple[str|None, str|None]
@@ -61,10 +62,14 @@ _runtime_vars: dict[str, Any] = {
     # registrar info
     '_registry_addrs': [],
 
-    '_is_infected_aio': False,
-
+    # `debug_mode: bool` settings
+    '_debug_mode': False,  # bool
+    'repl_fixture': False,  # |AbstractContextManager[bool]
     # for `tractor.pause_from_sync()` & `breakpoint()` support
     'use_greenback': False,
+
+    # infected-`asyncio`-mode: `trio` running as guest.
+    '_is_infected_aio': False,
 }
 
 

tractor/_streaming.py

@@ -426,8 +426,8 @@ class MsgStream(trio.abc.Channel):
             self._closed = re
 
         # if caught_eoc:
-        #     # from .devx import _debug
-        #     # await _debug.pause()
+        #     # from .devx import debug
+        #     # await debug.pause()
         #     with trio.CancelScope(shield=True):
         #         await rx_chan.aclose()
 

tractor/_supervise.py

@@ -21,7 +21,6 @@
 from contextlib import asynccontextmanager as acm
 from functools import partial
 import inspect
-from pprint import pformat
 from typing import (
     TYPE_CHECKING,
 )
@@ -31,7 +30,10 @@ import warnings
 import trio
 
 
-from .devx._debug import maybe_wait_for_debugger
+from .devx import (
+    debug,
+    pformat as _pformat,
+)
 from ._addr import (
     UnwrappedAddress,
     mk_uuid,
@@ -40,8 +42,11 @@ from ._state import current_actor, is_main_process
 from .log import get_logger, get_loglevel
 from ._runtime import Actor
 from ._portal import Portal
-from ._exceptions import (
+from .trionics import (
     is_multi_cancelled,
+    collapse_eg,
+)
+from ._exceptions import (
     ContextCancelled,
 )
 from ._root import (
@@ -112,7 +117,6 @@ class ActorNursery:
             ]
         ] = {}
 
-        self.cancelled: bool = False
         self._join_procs = trio.Event()
         self._at_least_one_child_in_debug: bool = False
         self.errors = errors
@@ -130,10 +134,53 @@ class ActorNursery:
         # TODO: remove the `.run_in_actor()` API and thus this 2ndary
         # nursery when that API get's moved outside this primitive!
         self._ria_nursery = ria_nursery
+
+        # TODO, factor this into a .hilevel api!
+        #
         # portals spawned with ``run_in_actor()`` are
         # cancelled when their "main" result arrives
         self._cancel_after_result_on_exit: set = set()
 
+        # trio.Nursery-like cancel (request) statuses
+        self._cancelled_caught: bool = False
+        self._cancel_called: bool = False
+
+    @property
+    def cancel_called(self) -> bool:
+        '''
+        Records whether cancellation has been requested for this
+        actor-nursery by a call to  `.cancel()` either due to,
+        - an explicit call by some actor-local-task,
+        - an implicit call due to an error/cancel emited inside
+          the `tractor.open_nursery()` block.
+
+        '''
+        return self._cancel_called
+
+    @property
+    def cancelled_caught(self) -> bool:
+        '''
+        Set when this nursery was able to cance all spawned subactors
+        gracefully via an (implicit) call to `.cancel()`.
+
+        '''
+        return self._cancelled_caught
+
+    # TODO! remove internal/test-suite usage!
+    @property
+    def cancelled(self) -> bool:
+        warnings.warn(
+            "`ActorNursery.cancelled` is now deprecated, use "
+            " `.cancel_called` instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return (
+            self._cancel_called
+            # and
+            # self._cancelled_caught
+        )
+
     async def start_actor(
         self,
         name: str,
@@ -197,7 +244,7 @@ class ActorNursery:
             loglevel=loglevel,
 
             # verbatim relay this actor's registrar addresses
-            registry_addrs=current_actor().reg_addrs,
+            registry_addrs=current_actor().registry_addrs,
         )
         parent_addr: UnwrappedAddress = self._actor.accept_addr
         assert parent_addr
@@ -311,7 +358,7 @@ class ActorNursery:
 
         '''
         __runtimeframe__: int = 1  # noqa
-        self.cancelled = True
+        self._cancel_called = True
 
         # TODO: impl a repr for spawn more compact
         # then `._children`..
@@ -322,9 +369,10 @@ class ActorNursery:
         server: IPCServer = self._actor.ipc_server
 
         with trio.move_on_after(3) as cs:
-            async with trio.open_nursery(
-                strict_exception_groups=False,
-            ) as tn:
+            async with (
+                collapse_eg(),
+                trio.open_nursery() as tn,
+            ):
 
                 subactor: Actor
                 proc: trio.Process
@@ -388,6 +436,8 @@ class ActorNursery:
             ) in children.values():
                 log.warning(f"Hard killing process {proc}")
                 proc.terminate()
+        else:
+            self._cancelled_caught
 
         # mark ourselves as having (tried to have) cancelled all subactors
         self._join_procs.set()
@@ -396,12 +446,12 @@ class ActorNursery:
 @acm
 async def _open_and_supervise_one_cancels_all_nursery(
     actor: Actor,
-    tb_hide: bool = False,
+    hide_tb: bool = True,
 
 ) -> typing.AsyncGenerator[ActorNursery, None]:
 
     # normally don't need to show user by default
-    __tracebackhide__: bool = tb_hide
+    __tracebackhide__: bool = hide_tb
 
     outer_err: BaseException|None = None
     inner_err: BaseException|None = None
@@ -417,10 +467,10 @@ async def _open_and_supervise_one_cancels_all_nursery(
     # `ActorNursery.start_actor()`).
 
     # errors from this daemon actor nursery bubble up to caller
-    async with trio.open_nursery(
-        strict_exception_groups=False,
-        # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
-    ) as da_nursery:
+    async with (
+        collapse_eg(),
+        trio.open_nursery() as da_nursery,
+    ):
         try:
             # This is the inner level "run in actor" nursery. It is
             # awaited first since actors spawned in this way (using
@@ -430,11 +480,10 @@ async def _open_and_supervise_one_cancels_all_nursery(
             # immediately raised for handling by a supervisor strategy.
             # As such if the strategy propagates any error(s) upwards
             # the above "daemon actor" nursery will be notified.
-            async with trio.open_nursery(
-                strict_exception_groups=False,
-                # ^XXX^ TODO? instead unpack any RAE as per "loose" style?
-            ) as ria_nursery:
-
+            async with (
+                collapse_eg(),
+                trio.open_nursery() as ria_nursery,
+            ):
                 an = ActorNursery(
                     actor,
                     ria_nursery,
@@ -451,7 +500,7 @@ async def _open_and_supervise_one_cancels_all_nursery(
                     # the "hard join phase".
                     log.runtime(
                         'Waiting on subactors to complete:\n'
-                        f'{pformat(an._children)}\n'
+                        f'>}} {len(an._children)}\n'
                     )
                     an._join_procs.set()
 
@@ -465,7 +514,7 @@ async def _open_and_supervise_one_cancels_all_nursery(
                     # will make the pdb repl unusable.
                     # Instead try to wait for pdb to be released before
                     # tearing down.
-                    await maybe_wait_for_debugger(
+                    await debug.maybe_wait_for_debugger(
                         child_in_debug=an._at_least_one_child_in_debug
                     )
 
@@ -541,7 +590,7 @@ async def _open_and_supervise_one_cancels_all_nursery(
 
             # XXX: yet another guard before allowing the cancel
             # sequence in case a (single) child is in debug.
-            await maybe_wait_for_debugger(
+            await debug.maybe_wait_for_debugger(
                 child_in_debug=an._at_least_one_child_in_debug
             )
 
@@ -590,9 +639,15 @@ async def _open_and_supervise_one_cancels_all_nursery(
     # final exit
 
 
+_shutdown_msg: str = (
+    'Actor-runtime-shutdown'
+)
+
+
 @acm
 # @api_frame
 async def open_nursery(
+    *,  # named params only!
     hide_tb: bool = True,
     **kwargs,
     # ^TODO, paramspec for `open_root_actor()`
@@ -677,17 +732,26 @@ async def open_nursery(
         ):
             __tracebackhide__: bool = False
 
-        msg: str = (
-            'Actor-nursery exited\n'
-            f'|_{an}\n'
+
+        op_nested_an_repr: str = _pformat.nest_from_op(
+            input_op=')>',
+            text=f'{an}',
+            # nest_prefix='|_',
+            nest_indent=1,  # under >
         )
+        an_msg: str = (
+            f'Actor-nursery exited\n'
+            f'{op_nested_an_repr}\n'
+        )
+        # keep noise low during std operation.
+        log.runtime(an_msg)
 
         if implicit_runtime:
             # shutdown runtime if it was started and report noisly
             # that we're did so.
-            msg += '=> Shutting down actor runtime <=\n'
+            msg: str = (
+                '\n'
+                '\n'
+                f'{_shutdown_msg} )>\n'
+            )
             log.info(msg)
-
-        else:
-            # keep noise low during std operation.
-            log.runtime(msg)

tractor/_testing/__init__.py

@@ -26,7 +26,7 @@ import os
 import pathlib
 
 import tractor
-from tractor.devx._debug import (
+from tractor.devx.debug import (
     BoxedMaybeException,
 )
 from .pytest import (
@@ -37,6 +37,9 @@ from .fault_simulation import (
 )
 
 
+# TODO, use dulwhich for this instead?
+# -> we're going to likely need it (or something similar)
+#   for supporting hot-coad reload feats eventually anyway!
 def repodir() -> pathlib.Path:
     '''
     Return the abspath to the repo directory.

tractor/devx/__init__.py

@@ -20,7 +20,7 @@ Runtime "developer experience" utils and addons to aid our
 and working with/on the actor runtime.
 
 """
-from ._debug import (
+from .debug import (
     maybe_wait_for_debugger as maybe_wait_for_debugger,
     acquire_debug_lock as acquire_debug_lock,
     breakpoint as breakpoint,

tractor/devx/_frame_stack.py

@@ -20,13 +20,18 @@ as it pertains to improving the grok-ability of our runtime!
 
 '''
 from __future__ import annotations
+from contextlib import (
+    _GeneratorContextManager,
+    _AsyncGeneratorContextManager,
+)
 from functools import partial
 import inspect
+import textwrap
 from types import (
     FrameType,
     FunctionType,
     MethodType,
-    # CodeType,
+    CodeType,
 )
 from typing import (
     Any,
@@ -34,6 +39,9 @@ from typing import (
     Type,
 )
 
+import pdbp
+from tractor.log import get_logger
+import trio
 from tractor.msg import (
     pretty_struct,
     NamespacePath,
@@ -41,6 +49,8 @@ from tractor.msg import (
 import wrapt
 
 
+log = get_logger(__name__)
+
 # TODO: yeah, i don't love this and we should prolly just
 # write a decorator that actually keeps a stupid ref to the func
 # obj..
@@ -301,3 +311,70 @@ def api_frame(
 #     error_set: set[BaseException],
 # ) -> TracebackType:
 #     ...
+
+
+def hide_runtime_frames() -> dict[FunctionType, CodeType]:
+    '''
+    Hide call-stack frames for various std-lib and `trio`-API primitives
+    such that the tracebacks presented from our runtime are as minimized
+    as possible, particularly from inside a `PdbREPL`.
+
+    '''
+    # XXX HACKZONE XXX
+    #  hide exit stack frames on nurseries and cancel-scopes!
+    # |_ so avoid seeing it when the `pdbp` REPL is first engaged from
+    #    inside a `trio.open_nursery()` scope (with no line after it
+    #    in before the block end??).
+    #
+    # TODO: FINALLY got this workin originally with
+    #  `@pdbp.hideframe` around the `wrapper()` def embedded inside
+    #  `_ki_protection_decoratior()`.. which is in the module:
+    #  /home/goodboy/.virtualenvs/tractor311/lib/python3.11/site-packages/trio/_core/_ki.py
+    #
+    # -[ ] make an issue and patch for `trio` core? maybe linked
+    #    to the long outstanding `pdb` one below?
+    #   |_ it's funny that there's frame hiding throughout `._run.py`
+    #      but not where it matters on the below exit funcs..
+    #
+    # -[ ] provide a patchset for the lonstanding
+    #   |_ https://github.com/python-trio/trio/issues/1155
+    #
+    # -[ ] make a linked issue to ^ and propose allowing all the
+    #     `._core._run` code to have their `__tracebackhide__` value
+    #     configurable by a `RunVar` to allow getting scheduler frames
+    #     if desired through configuration?
+    #
+    # -[ ] maybe dig into the core `pdb` issue why the extra frame is shown
+    #      at all?
+    #
+    funcs: list[FunctionType] = [
+        trio._core._run.NurseryManager.__aexit__,
+        trio._core._run.CancelScope.__exit__,
+         _GeneratorContextManager.__exit__,
+         _AsyncGeneratorContextManager.__aexit__,
+         _AsyncGeneratorContextManager.__aenter__,
+         trio.Event.wait,
+    ]
+    func_list_str: str = textwrap.indent(
+        "\n".join(f.__qualname__ for f in funcs),
+        prefix=' |_ ',
+    )
+    log.devx(
+        'Hiding the following runtime frames by default:\n'
+        f'{func_list_str}\n'
+    )
+
+    codes: dict[FunctionType, CodeType] = {}
+    for ref in funcs:
+        # stash a pre-modified version of each ref's code-obj
+        # so it can be reverted later if needed.
+        codes[ref] = ref.__code__
+        pdbp.hideframe(ref)
+    #
+    # pdbp.hideframe(trio._core._run.NurseryManager.__aexit__)
+    # pdbp.hideframe(trio._core._run.CancelScope.__exit__)
+    # pdbp.hideframe(_GeneratorContextManager.__exit__)
+    # pdbp.hideframe(_AsyncGeneratorContextManager.__aexit__)
+    # pdbp.hideframe(_AsyncGeneratorContextManager.__aenter__)
+    # pdbp.hideframe(trio.Event.wait)
+    return codes

tractor/devx/_stackscope.py

@@ -49,7 +49,7 @@ from tractor import (
     _state,
     log as logmod,
 )
-from tractor.devx import _debug
+from tractor.devx import debug
 
 log = logmod.get_logger(__name__)
 
@@ -82,7 +82,7 @@ def dump_task_tree() -> None:
     if (
         current_sigint_handler
         is not
-        _debug.DebugStatus._trio_handler
+        debug.DebugStatus._trio_handler
     ):
         sigint_handler_report: str = (
             'The default `trio` SIGINT handler was replaced?!'
@@ -237,8 +237,9 @@ def enable_stack_on_sig(
     try:
         import stackscope
     except ImportError:
-        log.error(
-            '`stackscope` not installed for use in debug mode!'
+        log.warning(
+            'The `stackscope` lib is not installed!\n'
+            '`Ignoring enable_stack_on_sig() call!\n'
         )
         return None
 

tractor/devx/debug/__init__.py

@@ -0,0 +1,100 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+Multi-actor debugging for da peeps!
+
+'''
+from __future__ import annotations
+from tractor.log import get_logger
+from ._repl import (
+    PdbREPL as PdbREPL,
+    mk_pdb as mk_pdb,
+    TractorConfig as TractorConfig,
+)
+from ._tty_lock import (
+    DebugStatus as DebugStatus,
+    DebugStateError as DebugStateError,
+)
+from ._trace import (
+    Lock as Lock,
+    _pause_msg as _pause_msg,
+    _repl_fail_msg as _repl_fail_msg,
+    _set_trace as _set_trace,
+    _sync_pause_from_builtin as _sync_pause_from_builtin,
+    breakpoint as breakpoint,
+    maybe_init_greenback as maybe_init_greenback,
+    maybe_import_greenback as maybe_import_greenback,
+    pause as pause,
+    pause_from_sync as pause_from_sync,
+)
+from ._post_mortem import (
+    BoxedMaybeException as BoxedMaybeException,
+    maybe_open_crash_handler as maybe_open_crash_handler,
+    open_crash_handler as open_crash_handler,
+    post_mortem as post_mortem,
+    _crash_msg as _crash_msg,
+    _maybe_enter_pm as _maybe_enter_pm,
+)
+from ._sync import (
+    maybe_wait_for_debugger as maybe_wait_for_debugger,
+    acquire_debug_lock as acquire_debug_lock,
+)
+from ._sigint import (
+    sigint_shield as sigint_shield,
+    _ctlc_ignore_header as _ctlc_ignore_header
+)
+
+log = get_logger(__name__)
+
+# ----------------
+# XXX PKG TODO XXX
+# ----------------
+# refine the internal impl and APIs!
+#
+# -[ ] rework `._pause()` and it's branch-cases for root vs.
+#     subactor:
+#  -[ ] `._pause_from_root()` + `_pause_from_subactor()`?
+#  -[ ]  do the de-factor based on bg-thread usage in
+#    `.pause_from_sync()` & `_pause_from_bg_root_thread()`.
+#  -[ ] drop `debug_func == None` case which is confusing af..
+#  -[ ]  factor out `_enter_repl_sync()` into a util func for calling
+#    the `_set_trace()` / `_post_mortem()` APIs?
+#
+# -[ ] figure out if we need `acquire_debug_lock()` and/or re-implement
+#    it as part of the `.pause_from_sync()` rework per above?
+#
+# -[ ] pair the `._pause_from_subactor()` impl with a "debug nursery"
+#   that's dynamically allocated inside the `._rpc` task thus
+#   avoiding the `._service_n.start()` usage for the IPC request?
+#  -[ ] see the TODO inside `._rpc._errors_relayed_via_ipc()`
+#
+# -[ ] impl a `open_debug_request()` which encaps all
+#   `request_root_stdio_lock()` task scheduling deats
+#   + `DebugStatus` state mgmt; which should prolly be re-branded as
+#   a `DebugRequest` type anyway AND with suppoort for bg-thread
+#   (from root actor) usage?
+#
+# -[ ] handle the `xonsh` case for bg-root-threads in the SIGINT
+#     handler!
+#   -[ ] do we need to do the same for subactors?
+#   -[ ] make the failing tests finally pass XD
+#
+# -[ ] simplify `maybe_wait_for_debugger()` to be a root-task only
+#     API?
+#   -[ ] currently it's implemented as that so might as well make it
+#     formal?

tractor/devx/debug/_post_mortem.py

@@ -0,0 +1,412 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+Post-mortem debugging APIs and surrounding machinery for both
+sync and async contexts.
+
+Generally we maintain the same semantics a `pdb.post.mortem()` but
+with actor-tree-wide sync/cooperation around any (sub)actor's use of
+the root's TTY.
+
+'''
+from __future__ import annotations
+import bdb
+from contextlib import (
+    AbstractContextManager,
+    contextmanager as cm,
+    nullcontext,
+)
+from functools import (
+    partial,
+)
+import inspect
+import sys
+import traceback
+from typing import (
+    Callable,
+    Sequence,
+    Type,
+    TYPE_CHECKING,
+)
+from types import (
+    TracebackType,
+    FrameType,
+)
+
+from msgspec import Struct
+import trio
+from tractor._exceptions import (
+    NoRuntime,
+)
+from tractor import _state
+from tractor._state import (
+    current_actor,
+    debug_mode,
+)
+from tractor.log import get_logger
+from tractor.trionics import (
+    is_multi_cancelled,
+)
+from ._trace import (
+    _pause,
+)
+from ._tty_lock import (
+    DebugStatus,
+)
+from ._repl import (
+    PdbREPL,
+    mk_pdb,
+    TractorConfig as TractorConfig,
+)
+
+if TYPE_CHECKING:
+    from trio.lowlevel import Task
+    from tractor._runtime import (
+        Actor,
+    )
+
+_crash_msg: str = (
+    'Opening a pdb REPL in crashed actor'
+)
+
+log = get_logger(__package__)
+
+
+class BoxedMaybeException(Struct):
+    '''
+    Box a maybe-exception for post-crash introspection usage
+    from the body of a `open_crash_handler()` scope.
+
+    '''
+    value: BaseException|None = None
+
+    # handler can suppress crashes dynamically
+    raise_on_exit: bool|Sequence[Type[BaseException]] = True
+
+    def pformat(self) -> str:
+        '''
+        Repr the boxed `.value` error in more-than-string
+        repr form.
+
+        '''
+        if not self.value:
+            return f'<{type(self).__name__}( .value=None )>'
+
+        return (
+            f'<{type(self.value).__name__}(\n'
+            f' |_.value = {self.value}\n'
+            f')>\n'
+        )
+
+    __repr__ = pformat
+
+
+def _post_mortem(
+    repl: PdbREPL,  # normally passed by `_pause()`
+
+    # XXX all `partial`-ed in by `post_mortem()` below!
+    tb: TracebackType,
+    api_frame: FrameType,
+
+    shield: bool = False,
+    hide_tb: bool = True,
+
+    # maybe pre/post REPL entry
+    repl_fixture: (
+        AbstractContextManager[bool]
+        |None
+    ) = None,
+
+    boxed_maybe_exc: BoxedMaybeException|None = None,
+
+) -> None:
+    '''
+    Enter the ``pdbpp`` port mortem entrypoint using our custom
+    debugger instance.
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    # maybe enter any user fixture
+    enter_repl: bool = DebugStatus.maybe_enter_repl_fixture(
+        repl=repl,
+        repl_fixture=repl_fixture,
+        boxed_maybe_exc=boxed_maybe_exc,
+    )
+    try:
+        if not enter_repl:
+            # XXX, trigger `.release()` below immediately!
+            return
+        try:
+            actor: Actor = current_actor()
+            actor_repr: str = str(actor.uid)
+            # ^TODO, instead a nice runtime-info + maddr + uid?
+            # -[ ] impl a `Actor.__repr()__`??
+            #  |_ <task>:<thread> @ <actor>
+
+        except NoRuntime:
+            actor_repr: str = '<no-actor-runtime?>'
+
+        try:
+            task_repr: Task = trio.lowlevel.current_task()
+        except RuntimeError:
+            task_repr: str = '<unknown-Task>'
+
+        # TODO: print the actor supervion tree up to the root
+        # here! Bo
+        log.pdb(
+            f'{_crash_msg}\n'
+            f'x>(\n'
+            f' |_ {task_repr} @ {actor_repr}\n'
+
+        )
+
+        # XXX NOTE(s) on `pdbp.xpm()` version..
+        #
+        # - seems to lose the up-stack tb-info?
+        # - currently we're (only) replacing this from `pdbp.xpm()`
+        #   to add the `end=''` to the print XD
+        #
+        print(traceback.format_exc(), end='')
+        caller_frame: FrameType = api_frame.f_back
+
+        # NOTE, see the impl details of these in the lib to
+        # understand usage:
+        # - `pdbp.post_mortem()`
+        # - `pdbp.xps()`
+        # - `bdb.interaction()`
+        repl.reset()
+        repl.interaction(
+            frame=caller_frame,
+            # frame=None,
+            traceback=tb,
+        )
+    finally:
+        # XXX NOTE XXX: this is abs required to avoid hangs!
+        #
+        # Since we presume the post-mortem was enaged to
+        # a task-ending error, we MUST release the local REPL request
+        # so that not other local task nor the root remains blocked!
+        DebugStatus.release()
+
+
+async def post_mortem(
+    *,
+    tb: TracebackType|None = None,
+    api_frame: FrameType|None = None,
+    hide_tb: bool = False,
+
+    # TODO: support shield here just like in `pause()`?
+    # shield: bool = False,
+
+    **_pause_kwargs,
+
+) -> None:
+    '''
+    Our builtin async equivalient of `pdb.post_mortem()` which can be
+    used inside exception handlers.
+
+    It's also used for the crash handler when `debug_mode == True` ;)
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    tb: TracebackType = tb or sys.exc_info()[2]
+
+    # TODO: do upward stack scan for highest @api_frame and
+    # use its parent frame as the expected user-app code
+    # interact point.
+    api_frame: FrameType = api_frame or inspect.currentframe()
+
+    # TODO, move to submod `._pausing` or ._api? _trace
+    await _pause(
+        debug_func=partial(
+            _post_mortem,
+            api_frame=api_frame,
+            tb=tb,
+        ),
+        hide_tb=hide_tb,
+        **_pause_kwargs
+    )
+
+
+async def _maybe_enter_pm(
+    err: BaseException,
+    *,
+    tb: TracebackType|None = None,
+    api_frame: FrameType|None = None,
+    hide_tb: bool = True,
+
+    # only enter debugger REPL when returns `True`
+    debug_filter: Callable[
+        [BaseException|BaseExceptionGroup],
+        bool,
+    ] = lambda err: not is_multi_cancelled(err),
+    **_pause_kws,
+):
+    if (
+        debug_mode()
+
+        # NOTE: don't enter debug mode recursively after quitting pdb
+        # Iow, don't re-enter the repl if the `quit` command was issued
+        # by the user.
+        and not isinstance(err, bdb.BdbQuit)
+
+        # XXX: if the error is the likely result of runtime-wide
+        # cancellation, we don't want to enter the debugger since
+        # there's races between when the parent actor has killed all
+        # comms and when the child tries to contact said parent to
+        # acquire the tty lock.
+
+        # Really we just want to mostly avoid catching KBIs here so there
+        # might be a simpler check we can do?
+        and
+        debug_filter(err)
+    ):
+        api_frame: FrameType = api_frame or inspect.currentframe()
+        tb: TracebackType = tb or sys.exc_info()[2]
+        await post_mortem(
+            api_frame=api_frame,
+            tb=tb,
+            **_pause_kws,
+        )
+        return True
+
+    else:
+        return False
+
+
+# TODO: better naming and what additionals?
+# - [ ] optional runtime plugging?
+# - [ ] detection for sync vs. async code?
+# - [ ] specialized REPL entry when in distributed mode?
+# -[x] hide tb by def
+# - [x] allow ignoring kbi Bo
+@cm
+def open_crash_handler(
+    catch: set[BaseException] = {
+        BaseException,
+    },
+    ignore: set[BaseException] = {
+        KeyboardInterrupt,
+        trio.Cancelled,
+    },
+    hide_tb: bool = True,
+
+    repl_fixture: (
+        AbstractContextManager[bool]  # pre/post REPL entry
+        |None
+    ) = None,
+    raise_on_exit: bool|Sequence[Type[BaseException]] = True,
+):
+    '''
+    Generic "post mortem" crash handler using `pdbp` REPL debugger.
+
+    We expose this as a CLI framework addon to both `click` and
+    `typer` users so they can quickly wrap cmd endpoints which get
+    automatically wrapped to use the runtime's `debug_mode: bool`
+    AND `pdbp.pm()` around any code that is PRE-runtime entry
+    - any sync code which runs BEFORE the main call to
+      `trio.run()`.
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    # TODO, yield a `outcome.Error`-like boxed type?
+    # -[~] use `outcome.Value/Error` X-> frozen!
+    # -[x] write our own..?
+    # -[ ] consider just wtv is used by `pytest.raises()`?
+    #
+    boxed_maybe_exc = BoxedMaybeException(
+        raise_on_exit=raise_on_exit,
+    )
+    err: BaseException
+    try:
+        yield boxed_maybe_exc
+    except tuple(catch) as err:
+        boxed_maybe_exc.value = err
+        if (
+            type(err) not in ignore
+            and
+            not is_multi_cancelled(
+                err,
+                ignore_nested=ignore
+            )
+        ):
+            try:
+                # use our re-impl-ed version of `pdbp.xpm()`
+                _post_mortem(
+                    repl=mk_pdb(),
+                    tb=sys.exc_info()[2],
+                    api_frame=inspect.currentframe().f_back,
+                    hide_tb=hide_tb,
+
+                    repl_fixture=repl_fixture,
+                    boxed_maybe_exc=boxed_maybe_exc,
+                )
+            except bdb.BdbQuit:
+                __tracebackhide__: bool = False
+                raise err
+
+        if (
+            raise_on_exit is True
+            or (
+                raise_on_exit is not False
+                and (
+                    set(raise_on_exit)
+                    and
+                    type(err) in raise_on_exit
+                )
+            )
+            and
+            boxed_maybe_exc.raise_on_exit == raise_on_exit
+        ):
+            raise err
+
+
+@cm
+def maybe_open_crash_handler(
+    pdb: bool|None = None,
+    hide_tb: bool = True,
+
+    **kwargs,
+):
+    '''
+    Same as `open_crash_handler()` but with bool input flag
+    to allow conditional handling.
+
+    Normally this is used with CLI endpoints such that if the --pdb
+    flag is passed the pdb REPL is engaed on any crashes B)
+
+    '''
+    __tracebackhide__: bool = hide_tb
+
+    if pdb is None:
+        pdb: bool = _state.is_debug_mode()
+
+    rtctx = nullcontext(
+        enter_result=BoxedMaybeException()
+    )
+    if pdb:
+        rtctx = open_crash_handler(
+            hide_tb=hide_tb,
+            **kwargs,
+        )
+
+    with rtctx as boxed_maybe_exc:
+        yield boxed_maybe_exc

tractor/devx/debug/_repl.py

@@ -0,0 +1,207 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+`pdpp.Pdb` extentions/customization and other delegate usage.
+
+'''
+from functools import (
+    cached_property,
+)
+import os
+
+import pdbp
+from tractor._state import (
+    is_root_process,
+)
+
+from ._tty_lock import (
+    Lock,
+    DebugStatus,
+)
+
+
+class TractorConfig(pdbp.DefaultConfig):
+    '''
+    Custom `pdbp` config which tries to use the best tradeoff
+    between pretty and minimal.
+
+    '''
+    use_pygments: bool = True
+    sticky_by_default: bool = False
+    enable_hidden_frames: bool = True
+
+    # much thanks @mdmintz for the hot tip!
+    # fixes line spacing issue when resizing terminal B)
+    truncate_long_lines: bool = False
+
+    # ------ - ------
+    # our own custom config vars mostly
+    # for syncing with the actor tree's singleton
+    # TTY `Lock`.
+
+
+class PdbREPL(pdbp.Pdb):
+    '''
+    Add teardown hooks and local state describing any
+    ongoing TTY `Lock` request dialog.
+
+    '''
+    # override the pdbp config with our coolio one
+    # NOTE: this is only loaded when no `~/.pdbrc` exists
+    # so we should prolly pass it into the .__init__() instead?
+    # i dunno, see the `DefaultFactory` and `pdb.Pdb` impls.
+    DefaultConfig = TractorConfig
+
+    status = DebugStatus
+
+    # NOTE: see details in stdlib's `bdb.py`
+    # def user_exception(self, frame, exc_info):
+    #     '''
+    #     Called when we stop on an exception.
+    #     '''
+    #     log.warning(
+    #         'Exception during REPL sesh\n\n'
+    #         f'{frame}\n\n'
+    #         f'{exc_info}\n\n'
+    #     )
+
+    # NOTE: this actually hooks but i don't see anyway to detect
+    # if an error was caught.. this is why currently we just always
+    # call `DebugStatus.release` inside `_post_mortem()`.
+    # def preloop(self):
+    #     print('IN PRELOOP')
+    #     super().preloop()
+
+    # TODO: cleaner re-wrapping of all this?
+    # -[ ] figure out how to disallow recursive .set_trace() entry
+    #     since that'll cause deadlock for us.
+    # -[ ] maybe a `@cm` to call `super().<same_meth_name>()`?
+    # -[ ] look at hooking into the `pp` hook specially with our
+    #     own set of pretty-printers?
+    #    * `.pretty_struct.Struct.pformat()`
+    #    * `.pformat(MsgType.pld)`
+    #    * `.pformat(Error.tb_str)`?
+    #    * .. maybe more?
+    #
+    def set_continue(self):
+        try:
+            super().set_continue()
+        finally:
+            # NOTE: for subactors the stdio lock is released via the
+            # allocated RPC locker task, so for root we have to do it
+            # manually.
+            if (
+                is_root_process()
+                and
+                Lock._debug_lock.locked()
+                and
+                DebugStatus.is_main_trio_thread()
+            ):
+                # Lock.release(raise_on_thread=False)
+                Lock.release()
+
+            # XXX AFTER `Lock.release()` for root local repl usage
+            DebugStatus.release()
+
+    def set_quit(self):
+        try:
+            super().set_quit()
+        finally:
+            if (
+                is_root_process()
+                and
+                Lock._debug_lock.locked()
+                and
+                DebugStatus.is_main_trio_thread()
+            ):
+                # Lock.release(raise_on_thread=False)
+                Lock.release()
+
+            # XXX after `Lock.release()` for root local repl usage
+            DebugStatus.release()
+
+    # XXX NOTE: we only override this because apparently the stdlib pdb
+    # bois likes to touch the SIGINT handler as much as i like to touch
+    # my d$%&.
+    def _cmdloop(self):
+        self.cmdloop()
+
+    @cached_property
+    def shname(self) -> str | None:
+        '''
+        Attempt to return the login shell name with a special check for
+        the infamous `xonsh` since it seems to have some issues much
+        different from std shells when it comes to flushing the prompt?
+
+        '''
+        # SUPER HACKY and only really works if `xonsh` is not used
+        # before spawning further sub-shells..
+        shpath = os.getenv('SHELL', None)
+
+        if shpath:
+            if (
+                os.getenv('XONSH_LOGIN', default=False)
+                or 'xonsh' in shpath
+            ):
+                return 'xonsh'
+
+            return os.path.basename(shpath)
+
+        return None
+
+
+def mk_pdb() -> PdbREPL:
+    '''
+    Deliver a new `PdbREPL`: a multi-process safe `pdbp.Pdb`-variant
+    using the magic of `tractor`'s SC-safe IPC.
+
+    B)
+
+    Our `pdb.Pdb` subtype accomplishes multi-process safe debugging
+    by:
+
+    - mutexing access to the root process' std-streams (& thus parent
+      process TTY) via an IPC managed `Lock` singleton per
+      actor-process tree.
+
+    - temporarily overriding any subactor's SIGINT handler to shield
+      during live REPL sessions in sub-actors such that cancellation
+      is never (mistakenly) triggered by a ctrl-c and instead only by
+      explicit runtime API requests or after the
+      `pdb.Pdb.interaction()` call has returned.
+
+    FURTHER, the `pdbp.Pdb` instance is configured to be `trio`
+    "compatible" from a SIGINT handling perspective; we mask out
+    the default `pdb` handler and instead apply `trio`s default
+    which mostly addresses all issues described in:
+
+     - https://github.com/python-trio/trio/issues/1155
+
+    The instance returned from this factory should always be
+    preferred over the default `pdb[p].set_trace()` whenever using
+    a `pdb` REPL inside a `trio` based runtime.
+
+    '''
+    pdb = PdbREPL()
+
+    # XXX: These are the important flags mentioned in
+    # https://github.com/python-trio/trio/issues/1155
+    # which resolve the traceback spews to console.
+    pdb.allow_kbdint = True
+    pdb.nosigint = True
+    return pdb

tractor/devx/debug/_sigint.py

@@ -0,0 +1,333 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+A custom SIGINT handler which mainly shields actor (task)
+cancellation during REPL interaction.
+
+'''
+from __future__ import annotations
+from typing import (
+    TYPE_CHECKING,
+)
+import trio
+from tractor.log import get_logger
+from tractor._state import (
+    current_actor,
+    is_root_process,
+)
+from ._repl import (
+    PdbREPL,
+)
+from ._tty_lock import (
+    any_connected_locker_child,
+    DebugStatus,
+    Lock,
+)
+
+if TYPE_CHECKING:
+    from tractor.ipc import (
+        Channel,
+    )
+    from tractor._runtime import (
+        Actor,
+    )
+
+log = get_logger(__name__)
+
+_ctlc_ignore_header: str = (
+    'Ignoring SIGINT while debug REPL in use'
+)
+
+
+def sigint_shield(
+    signum: int,
+    frame: 'frame',  # type: ignore # noqa
+    *args,
+
+) -> None:
+    '''
+    Specialized, debugger-aware SIGINT handler.
+
+    In childred we always ignore/shield for SIGINT to avoid
+    deadlocks since cancellation should always be managed by the
+    supervising parent actor. The root actor-proces is always
+    cancelled on ctrl-c.
+
+    '''
+    __tracebackhide__: bool = True
+    actor: Actor = current_actor()
+
+    def do_cancel():
+        # If we haven't tried to cancel the runtime then do that instead
+        # of raising a KBI (which may non-gracefully destroy
+        # a ``trio.run()``).
+        if not actor._cancel_called:
+            actor.cancel_soon()
+
+        # If the runtime is already cancelled it likely means the user
+        # hit ctrl-c again because teardown didn't fully take place in
+        # which case we do the "hard" raising of a local KBI.
+        else:
+            raise KeyboardInterrupt
+
+    # only set in the actor actually running the REPL
+    repl: PdbREPL|None = DebugStatus.repl
+
+    # TODO: maybe we should flatten out all these cases using
+    # a match/case?
+    #
+    # root actor branch that reports whether or not a child
+    # has locked debugger.
+    if is_root_process():
+        # log.warning(
+        log.devx(
+            'Handling SIGINT in root actor\n'
+            f'{Lock.repr()}'
+            f'{DebugStatus.repr()}\n'
+        )
+        # try to see if the supposed (sub)actor in debug still
+        # has an active connection to *this* actor, and if not
+        # it's likely they aren't using the TTY lock / debugger
+        # and we should propagate SIGINT normally.
+        any_connected: bool = any_connected_locker_child()
+
+        problem = (
+            f'root {actor.uid} handling SIGINT\n'
+            f'any_connected: {any_connected}\n\n'
+
+            f'{Lock.repr()}\n'
+        )
+
+        if (
+            (ctx := Lock.ctx_in_debug)
+            and
+            (uid_in_debug := ctx.chan.uid) # "someone" is (ostensibly) using debug `Lock`
+        ):
+            name_in_debug: str = uid_in_debug[0]
+            assert not repl
+            # if not repl:  # but it's NOT us, the root actor.
+            # sanity: since no repl ref is set, we def shouldn't
+            # be the lock owner!
+            assert name_in_debug != 'root'
+
+            # IDEAL CASE: child has REPL as expected
+            if any_connected:  # there are subactors we can contact
+                # XXX: only if there is an existing connection to the
+                # (sub-)actor in debug do we ignore SIGINT in this
+                # parent! Otherwise we may hang waiting for an actor
+                # which has already terminated to unlock.
+                #
+                # NOTE: don't emit this with `.pdb()` level in
+                # root without a higher level.
+                log.runtime(
+                    _ctlc_ignore_header
+                    +
+                    f' by child '
+                    f'{uid_in_debug}\n'
+                )
+                problem = None
+
+            else:
+                problem += (
+                    '\n'
+                    f'A `pdb` REPL is SUPPOSEDLY in use by child {uid_in_debug}\n'
+                    f'BUT, no child actors are IPC contactable!?!?\n'
+                )
+
+        # IDEAL CASE: root has REPL as expected
+        else:
+            # root actor still has this SIGINT handler active without
+            # an actor using the `Lock` (a bug state) ??
+            # => so immediately cancel any stale lock cs and revert
+            # the handler!
+            if not DebugStatus.repl:
+                # TODO: WHEN should we revert back to ``trio``
+                # handler if this one is stale?
+                # -[ ] maybe after a counts work of ctl-c mashes?
+                # -[ ] use a state var like `stale_handler: bool`?
+                problem += (
+                    'No subactor is using a `pdb` REPL according `Lock.ctx_in_debug`?\n'
+                    'BUT, the root should be using it, WHY this handler ??\n\n'
+                    'So either..\n'
+                    '- some root-thread is using it but has no `.repl` set?, OR\n'
+                    '- something else weird is going on outside the runtime!?\n'
+                )
+            else:
+                # NOTE: since we emit this msg on ctl-c, we should
+                # also always re-print the prompt the tail block!
+                log.pdb(
+                    _ctlc_ignore_header
+                    +
+                    f' by root actor..\n'
+                    f'{DebugStatus.repl_task}\n'
+                    f' |_{repl}\n'
+                )
+                problem = None
+
+        # XXX if one is set it means we ARE NOT operating an ideal
+        # case where a child subactor or us (the root) has the
+        # lock without any other detected problems.
+        if problem:
+
+            # detect, report and maybe clear a stale lock request
+            # cancel scope.
+            lock_cs: trio.CancelScope = Lock.get_locking_task_cs()
+            maybe_stale_lock_cs: bool = (
+                lock_cs is not None
+                and not lock_cs.cancel_called
+            )
+            if maybe_stale_lock_cs:
+                problem += (
+                    '\n'
+                    'Stale `Lock.ctx_in_debug._scope: CancelScope` detected?\n'
+                    f'{Lock.ctx_in_debug}\n\n'
+
+                    '-> Calling ctx._scope.cancel()!\n'
+                )
+                lock_cs.cancel()
+
+            # TODO: wen do we actually want/need this, see above.
+            # DebugStatus.unshield_sigint()
+            log.warning(problem)
+
+    # child actor that has locked the debugger
+    elif not is_root_process():
+        log.debug(
+            f'Subactor {actor.uid} handling SIGINT\n\n'
+            f'{Lock.repr()}\n'
+        )
+
+        rent_chan: Channel = actor._parent_chan
+        if (
+            rent_chan is None
+            or
+            not rent_chan.connected()
+        ):
+            log.warning(
+                'This sub-actor thinks it is debugging '
+                'but it has no connection to its parent ??\n'
+                f'{actor.uid}\n'
+                'Allowing SIGINT propagation..'
+            )
+            DebugStatus.unshield_sigint()
+
+        repl_task: str|None = DebugStatus.repl_task
+        req_task: str|None = DebugStatus.req_task
+        if (
+            repl_task
+            and
+            repl
+        ):
+            log.pdb(
+                _ctlc_ignore_header
+                +
+                f' by local task\n\n'
+                f'{repl_task}\n'
+                f' |_{repl}\n'
+            )
+        elif req_task:
+            log.debug(
+                _ctlc_ignore_header
+                +
+                f' by local request-task and either,\n'
+                f'- someone else is already REPL-in and has the `Lock`, or\n'
+                f'- some other local task already is replin?\n\n'
+                f'{req_task}\n'
+            )
+
+        # TODO can we remove this now?
+        # -[ ] does this path ever get hit any more?
+        else:
+            msg: str = (
+                'SIGINT shield handler still active BUT, \n\n'
+            )
+            if repl_task is None:
+                msg += (
+                    '- No local task claims to be in debug?\n'
+                )
+
+            if repl is None:
+                msg += (
+                    '- No local REPL is currently active?\n'
+                )
+
+            if req_task is None:
+                msg += (
+                    '- No debug request task is active?\n'
+                )
+
+            log.warning(
+                msg
+                +
+                'Reverting handler to `trio` default!\n'
+            )
+            DebugStatus.unshield_sigint()
+
+            # XXX ensure that the reverted-to-handler actually is
+            # able to rx what should have been **this** KBI ;)
+            do_cancel()
+
+        # TODO: how to handle the case of an intermediary-child actor
+        # that **is not** marked in debug mode? See oustanding issue:
+        # https://github.com/goodboy/tractor/issues/320
+        # elif debug_mode():
+
+    # maybe redraw/print last REPL output to console since
+    # we want to alert the user that more input is expect since
+    # nothing has been done dur to ignoring sigint.
+    if (
+        DebugStatus.repl  # only when current actor has a REPL engaged
+    ):
+        flush_status: str = (
+            'Flushing stdout to ensure new prompt line!\n'
+        )
+
+        # XXX: yah, mega hack, but how else do we catch this madness XD
+        if (
+            repl.shname == 'xonsh'
+        ):
+            flush_status += (
+                '-> ALSO re-flushing due to `xonsh`..\n'
+            )
+            repl.stdout.write(repl.prompt)
+
+        # log.warning(
+        log.devx(
+            flush_status
+        )
+        repl.stdout.flush()
+
+        # TODO: better console UX to match the current "mode":
+        # -[ ] for example if in sticky mode where if there is output
+        #   detected as written to the tty we redraw this part underneath
+        #   and erase the past draw of this same bit above?
+        # repl.sticky = True
+        # repl._print_if_sticky()
+
+        # also see these links for an approach from `ptk`:
+        # https://github.com/goodboy/tractor/issues/130#issuecomment-663752040
+        # https://github.com/prompt-toolkit/python-prompt-toolkit/blob/c2c6af8a0308f9e5d7c0e28cb8a02963fe0ce07a/prompt_toolkit/patch_stdout.py
+    else:
+        log.devx(
+        # log.warning(
+            'Not flushing stdout since not needed?\n'
+            f'|_{repl}\n'
+        )
+
+    # XXX only for tracing this handler
+    log.devx('exiting SIGINT')

tractor/devx/debug/_sync.py

@@ -0,0 +1,220 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public
+# License along with this program.  If not, see
+# <https://www.gnu.org/licenses/>.
+
+'''
+Debugger synchronization APIs to ensure orderly access and
+non-TTY-clobbering graceful teardown.
+
+
+'''
+from __future__ import annotations
+from contextlib import (
+    asynccontextmanager as acm,
+)
+from functools import (
+    partial,
+)
+from typing import (
+    AsyncGenerator,
+    Callable,
+)
+
+from tractor.log import get_logger
+import trio
+from trio.lowlevel import (
+    current_task,
+    Task,
+)
+from tractor._context import Context
+from tractor._state import (
+    current_actor,
+    debug_mode,
+    is_root_process,
+)
+from ._repl import (
+    TractorConfig as TractorConfig,
+)
+from ._tty_lock import (
+    Lock,
+    request_root_stdio_lock,
+    any_connected_locker_child,
+)
+from ._sigint import (
+    sigint_shield as sigint_shield,
+    _ctlc_ignore_header as _ctlc_ignore_header
+)
+
+log = get_logger(__package__)
+
+
+async def maybe_wait_for_debugger(
+    poll_steps: int = 2,
+    poll_delay: float = 0.1,
+    child_in_debug: bool = False,
+
+    header_msg: str = '',
+    _ll: str = 'devx',
+
+) -> bool:  # was locked and we polled?
+
+    if (
+        not debug_mode()
+        and
+        not child_in_debug
+    ):
+        return False
+
+    logmeth: Callable = getattr(log, _ll)
+
+    msg: str = header_msg
+    if (
+        is_root_process()
+    ):
+        # If we error in the root but the debugger is
+        # engaged we don't want to prematurely kill (and
+        # thus clobber access to) the local tty since it
+        # will make the pdb repl unusable.
+        # Instead try to wait for pdb to be released before
+        # tearing down.
+        ctx_in_debug: Context|None = Lock.ctx_in_debug
+        in_debug: tuple[str, str]|None = (
+            ctx_in_debug.chan.uid
+            if ctx_in_debug
+            else None
+        )
+        if in_debug == current_actor().uid:
+            log.debug(
+                msg
+                +
+                'Root already owns the TTY LOCK'
+            )
+            return True
+
+        elif in_debug:
+            msg += (
+                f'Debug `Lock` in use by subactor\n|\n|_{in_debug}\n'
+            )
+            # TODO: could this make things more deterministic?
+            # wait to see if a sub-actor task will be
+            # scheduled and grab the tty lock on the next
+            # tick?
+            # XXX => but it doesn't seem to work..
+            # await trio.testing.wait_all_tasks_blocked(cushion=0)
+        else:
+            logmeth(
+                msg
+                +
+                'Root immediately acquired debug TTY LOCK'
+            )
+            return False
+
+        for istep in range(poll_steps):
+            if (
+                Lock.req_handler_finished is not None
+                and not Lock.req_handler_finished.is_set()
+                and in_debug is not None
+            ):
+                # caller_frame_info: str = pformat_caller_frame()
+                logmeth(
+                    msg
+                    +
+                    '\n^^ Root is waiting on tty lock release.. ^^\n'
+                    # f'{caller_frame_info}\n'
+                )
+
+                if not any_connected_locker_child():
+                    Lock.get_locking_task_cs().cancel()
+
+                with trio.CancelScope(shield=True):
+                    await Lock.req_handler_finished.wait()
+
+                log.devx(
+                    f'Subactor released debug lock\n'
+                    f'|_{in_debug}\n'
+                )
+                break
+
+            # is no subactor locking debugger currently?
+            if (
+                in_debug is None
+                and (
+                    Lock.req_handler_finished is None
+                    or Lock.req_handler_finished.is_set()
+                )
+            ):
+                logmeth(
+                    msg
+                    +
+                    'Root acquired tty lock!'
+                )
+                break
+
+            else:
+                logmeth(
+                    'Root polling for debug:\n'
+                    f'poll step: {istep}\n'
+                    f'poll delya: {poll_delay}\n\n'
+                    f'{Lock.repr()}\n'
+                )
+                with trio.CancelScope(shield=True):
+                    await trio.sleep(poll_delay)
+                    continue
+
+        return True
+
+    # else:
+    #     # TODO: non-root call for #320?
+    #     this_uid: tuple[str, str] = current_actor().uid
+    #     async with acquire_debug_lock(
+    #         subactor_uid=this_uid,
+    #     ):
+    #         pass
+    return False
+
+
+@acm
+async def acquire_debug_lock(
+    subactor_uid: tuple[str, str],
+) -> AsyncGenerator[
+    trio.CancelScope|None,
+    tuple,
+]:
+    '''
+    Request to acquire the TTY `Lock` in the root actor, release on
+    exit.
+
+    This helper is for actor's who don't actually need to acquired
+    the debugger but want to wait until the lock is free in the
+    process-tree root such that they don't clobber an ongoing pdb
+    REPL session in some peer or child!
+
+    '''
+    if not debug_mode():
+        yield None
+        return
+
+    task: Task = current_task()
+    async with trio.open_nursery() as n:
+        ctx: Context = await n.start(
+            partial(
+                request_root_stdio_lock,
+                actor_uid=subactor_uid,
+                task_uid=(task.name, id(task)),
+            )
+        )
+        yield ctx
+        ctx.cancel()

tractor/devx/pformat.py

@@ -15,8 +15,10 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 '''
-Pretty formatters for use throughout the code base.
-Mostly handy for logging and exception message content.
+Pretty formatters for use throughout our internals.
+
+Handy for logging and exception message content but also for `repr()`
+in REPL(s).
 
 '''
 import sys
@@ -224,8 +226,8 @@ def pformat_cs(
     field_prefix: str = ' |_',
 ) -> str:
     '''
-    Pretty format info about a `trio.CancelScope` including most
-    of its public state and `._cancel_status`.
+    Pretty format info about a `trio.CancelScope` including most of
+    its public state and `._cancel_status`.
 
     The output can be modified to show a "var name" for the
     instance as a field prefix, just a simple str before each
@@ -247,3 +249,279 @@ def pformat_cs(
         +
         fields
     )
+
+
+def nest_from_op(
+    input_op: str,  # TODO, Literal of all op-"symbols" from below?
+    text: str,
+    prefix_op: bool = True,  # unset is to suffix the first line
+    # optionally suffix `text`, by def on a newline
+    op_suffix='\n',
+
+    nest_prefix: str = '|_',
+    nest_indent: int|None = None,
+    # XXX indent `next_prefix` "to-the-right-of" `input_op`
+    # by this count of whitespaces (' ').
+    rm_from_first_ln: str|None = None,
+
+) -> str:
+    '''
+    Depth-increment the input (presumably hierarchy/supervision)
+    input "tree string" below the provided `input_op` execution
+    operator, so injecting a `"\n|_{input_op}\n"`and indenting the
+    `tree_str` to nest content aligned with the ops last char.
+
+    '''
+    # `sclang` "structurred-concurrency-language": an ascii-encoded
+    # symbolic alphabet to describe concurrent systems.
+    #
+    # ?TODO? aa more fomal idea for a syntax to the state of
+    # concurrent systems as a "3-domain" (execution, scope, storage)
+    # model and using a minimal ascii/utf-8 operator-set.
+    #
+    # try not to take any of this seriously yet XD
+    #
+    # > is a "play operator" indicating (CPU bound)
+    #   exec/work/ops required at the "lowest level computing"
+    #
+    # execution primititves (tasks, threads, actors..) denote their
+    # lifetime with '(' and ')' since parentheses normally are used
+    # in many langs to denote function calls.
+    #
+    # starting = (
+    # >(  opening/starting; beginning of the thread-of-exec (toe?)
+    # (>  opened/started,  (finished spawning toe)
+    # |_<Task: blah blah..>  repr of toe, in py these look like <objs>
+    #
+    # >) closing/exiting/stopping,
+    # )> closed/exited/stopped,
+    # |_<Task: blah blah..>
+    #   [OR <), )< ?? ]
+    #
+    # ending = )
+    # >c) cancelling to close/exit
+    # c)> cancelled (caused close), OR?
+    #  |_<Actor: ..>
+    #   OR maybe "<c)" which better indicates the cancel being
+    #   "delivered/returned" / returned" to LHS?
+    #
+    # >x)  erroring to eventuall exit
+    # x)>  errored and terminated
+    #  |_<Actor: ...>
+    #
+    # scopes: supers/nurseries, IPC-ctxs, sessions, perms, etc.
+    # >{  opening
+    # {>  opened
+    # }>  closed
+    # >}  closing
+    #
+    # storage: like queues, shm-buffers, files, etc..
+    # >[  opening
+    # [>  opened
+    #  |_<FileObj: ..>
+    #
+    # >]  closing
+    # ]>  closed
+
+    # IPC ops: channels, transports, msging
+    # =>  req msg
+    # <=  resp msg
+    # <=> 2-way streaming (of msgs)
+    # <-  recv 1 msg
+    # ->  send 1 msg
+    #
+    # TODO: still not sure on R/L-HS approach..?
+    # =>(  send-req to exec start (task, actor, thread..)
+    # (<=  recv-req to ^
+    #
+    # (<=  recv-req ^
+    # <=(  recv-resp opened remote exec primitive
+    # <=)  recv-resp closed
+    #
+    # )<=c req to stop due to cancel
+    # c=>) req to stop due to cancel
+    #
+    # =>{  recv-req to open
+    # <={  send-status that it closed
+    #
+    if (
+        nest_prefix
+        and
+        nest_indent != 0
+    ):
+        if nest_indent is not None:
+            nest_prefix: str = textwrap.indent(
+                nest_prefix,
+                prefix=nest_indent*' ',
+            )
+        nest_indent: int = len(nest_prefix)
+
+    # determine body-text indent either by,
+    # - using wtv explicit indent value is provided,
+    # OR
+    # - auto-calcing the indent to embed `text` under
+    #   the `nest_prefix` if provided, **IFF** `nest_indent=None`.
+    tree_str_indent: int = 0
+    if nest_indent not in {0, None}:
+        tree_str_indent = nest_indent
+    elif (
+        nest_prefix
+        and
+        nest_indent != 0
+    ):
+        tree_str_indent = len(nest_prefix)
+
+    indented_tree_str: str = text
+    if tree_str_indent:
+        indented_tree_str: str = textwrap.indent(
+            text,
+            prefix=' '*tree_str_indent,
+        )
+
+    # inject any provided nesting-prefix chars
+    # into the head of the first line.
+    if nest_prefix:
+        indented_tree_str: str = (
+            f'{nest_prefix}{indented_tree_str[tree_str_indent:]}'
+        )
+
+    if (
+        not prefix_op
+        or
+        rm_from_first_ln
+    ):
+        tree_lns: list[str] = indented_tree_str.splitlines()
+        first: str = tree_lns[0]
+        if rm_from_first_ln:
+            first = first.strip().replace(
+                rm_from_first_ln,
+                '',
+            )
+        indented_tree_str: str = '\n'.join(tree_lns[1:])
+
+        if prefix_op:
+            indented_tree_str = (
+                f'{first}\n'
+                f'{indented_tree_str}'
+            )
+
+    if prefix_op:
+        return (
+            f'{input_op}{op_suffix}'
+            f'{indented_tree_str}'
+        )
+    else:
+        return (
+            f'{first}{input_op}{op_suffix}'
+            f'{indented_tree_str}'
+        )
+
+
+# ------ modden.repr ------
+# XXX originally taken verbaatim from `modden.repr`
+'''
+More "multi-line" representation then the stdlib's `pprint` equivs.
+
+'''
+from inspect import (
+    FrameInfo,
+    stack,
+)
+import pprint
+import reprlib
+from typing import (
+    Callable,
+)
+
+
+def mk_repr(
+    **repr_kws,
+) -> Callable[[str], str]:
+    '''
+    Allocate and deliver a `repr.Repr` instance with provided input
+    settings using the std-lib's `reprlib` mod,
+     * https://docs.python.org/3/library/reprlib.html
+
+    ------ Ex. ------
+    An up to 6-layer-nested `dict` as multi-line:
+    - https://stackoverflow.com/a/79102479
+    - https://docs.python.org/3/library/reprlib.html#reprlib.Repr.maxlevel
+
+    '''
+    def_kws: dict[str, int] = dict(
+        indent=3,  # indent used for repr of recursive objects
+        maxlevel=616,  # recursion levels
+        maxdict=616,  # max items shown for `dict`
+        maxlist=616,  # max items shown for `dict`
+        maxstring=616,  # match editor line-len limit
+        maxtuple=616,  # match editor line-len limit
+        maxother=616,  # match editor line-len limit
+    )
+    def_kws |= repr_kws
+    reprr = reprlib.Repr(**def_kws)
+    return reprr.repr
+
+
+def ppfmt(
+    obj: object,
+    do_print: bool = False,
+) -> str:
+    '''
+    The `pprint.pformat()` version of `pprint.pp()`, namely
+    a default `sort_dicts=False`.. (which i think should be
+    the normal default in the stdlib).
+
+    '''
+    pprepr: Callable = mk_repr()
+    repr_str: str = pprepr(obj)
+
+    if do_print:
+        return pprint.pp(repr_str)
+
+    return repr_str
+
+
+pformat = ppfmt
+
+
+def pfmt_frame_info(fi: FrameInfo) -> str:
+    '''
+    Like a std `inspect.FrameInfo.__repr__()` but multi-line..
+
+    '''
+    return (
+        'FrameInfo(\n'
+        '  frame={!r},\n'
+        '  filename={!r},\n'
+        '  lineno={!r},\n'
+        '  function={!r},\n'
+        '  code_context={!r},\n'
+        '  index={!r},\n'
+        '  positions={!r})'
+        ).format(
+            fi.frame,
+            fi.filename,
+            fi.lineno,
+            fi.function,
+            fi.code_context,
+            fi.index,
+            fi.positions
+        )
+
+
+def pfmt_callstack(frames: int = 1) -> str:
+    '''
+    Generate a string of nested `inspect.FrameInfo` objects returned
+    from a `inspect.stack()` call such that only the `.frame` field
+    for each  layer is pprinted.
+
+    '''
+    caller_frames: list[FrameInfo] =  stack()[1:1+frames]
+    frames_str: str = ''
+    for i, frame_info in enumerate(caller_frames):
+        frames_str += textwrap.indent(
+            f'{frame_info.frame!r}\n',
+            prefix=' '*i,
+
+        )
+    return frames_str

tractor/ipc/_chan.py

@@ -101,11 +101,27 @@ class Channel:
         # ^XXX! ONLY set if a remote actor sends an `Error`-msg
         self._closed: bool = False
 
-        # flag set by ``Portal.cancel_actor()`` indicating remote
-        # (possibly peer) cancellation of the far end actor
-        # runtime.
+        # flag set by `Portal.cancel_actor()` indicating remote
+        # (possibly peer) cancellation of the far end actor runtime.
         self._cancel_called: bool = False
 
+    @property
+    def closed(self) -> bool:
+        '''
+        Was `.aclose()` successfully called?
+
+        '''
+        return self._closed
+
+    @property
+    def cancel_called(self) -> bool:
+        '''
+        Set when `Portal.cancel_actor()` is called on a portal which
+        wraps this IPC channel.
+
+        '''
+        return self._cancel_called
+
     @property
     def uid(self) -> tuple[str, str]:
         '''
@@ -169,13 +185,27 @@ class Channel:
             addr,
             **kwargs,
         )
-        assert transport.raddr == addr
+        # XXX, for UDS *no!* since we recv the peer-pid and build out
+        # a new addr..
+        # assert transport.raddr == addr
         chan = Channel(transport=transport)
-        log.runtime(
-            f'Connected channel IPC transport\n'
-            f'[>\n'
-            f' |_{chan}\n'
-        )
+
+        # ?TODO, compact this into adapter level-methods?
+        # -[ ] would avoid extra repr-calcs if level not active?
+        #   |_ how would the `calc_if_level` look though? func?
+        if log.at_least_level('runtime'):
+            from tractor.devx import (
+                pformat as _pformat,
+            )
+            chan_repr: str = _pformat.nest_from_op(
+                input_op='[>',
+                text=chan.pformat(),
+                nest_indent=1,
+            )
+            log.runtime(
+                f'Connected channel IPC transport\n'
+                f'{chan_repr}'
+            )
         return chan
 
     @cm
@@ -196,9 +226,12 @@ class Channel:
             self._transport.codec = orig
 
     # TODO: do a .src/.dst: str for maddrs?
-    def pformat(self) -> str:
+    def pformat(
+        self,
+        privates: bool = False,
+    ) -> str:
         if not self._transport:
-            return '<Channel with inactive transport?>'
+            return '<Channel( with inactive transport? )>'
 
         tpt: MsgTransport = self._transport
         tpt_name: str = type(tpt).__name__
@@ -206,26 +239,35 @@ class Channel:
             'connected' if self.connected()
             else 'closed'
         )
-        return (
+        repr_str: str = (
             f'<Channel(\n'
             f' |_status: {tpt_status!r}\n'
+        ) + (
             f'   _closed={self._closed}\n'
             f'   _cancel_called={self._cancel_called}\n'
-            f'\n'
-            f' |_peer: {self.aid}\n'
-            f'\n'
+            if privates else ''
+        ) + (  # peer-actor (processs) section
+            f' |_peer: {self.aid.reprol()!r}\n'
+            if self.aid else ' |_peer: <unknown>\n'
+        ) + (
             f' |_msgstream: {tpt_name}\n'
-            f'   proto={tpt.laddr.proto_key!r}\n'
-            f'   layer={tpt.layer_key!r}\n'
-            f'   laddr={tpt.laddr}\n'
-            f'   raddr={tpt.raddr}\n'
-            f'   codec={tpt.codec_key!r}\n'
-            f'   stream={tpt.stream}\n'
-            f'   maddr={tpt.maddr!r}\n'
-            f'   drained={tpt.drained}\n'
+            f'   maddr: {tpt.maddr!r}\n'
+            f'   proto: {tpt.laddr.proto_key!r}\n'
+            f'   layer: {tpt.layer_key!r}\n'
+            f'   codec: {tpt.codec_key!r}\n'
+            f'   .laddr={tpt.laddr}\n'
+            f'   .raddr={tpt.raddr}\n'
+        ) + (
+            f'   ._transport.stream={tpt.stream}\n'
+            f'   ._transport.drained={tpt.drained}\n'
+            if privates else ''
+        ) + (
             f'   _send_lock={tpt._send_lock.statistics()}\n'
-            f')>\n'
+            if privates else ''
+        ) + (
+            ')>\n'
         )
+        return repr_str
 
     # NOTE: making this return a value that can be passed to
     # `eval()` is entirely **optional** FYI!
@@ -247,6 +289,10 @@ class Channel:
     def raddr(self) -> Address|None:
         return self._transport.raddr if self._transport else None
 
+    @property
+    def maddr(self) -> str:
+        return self._transport.maddr if self._transport else '<no-tpt>'
+
     # TODO: something like,
     # `pdbp.hideframe_on(errors=[MsgTypeError])`
     # instead of the `try/except` hack we have rn..
@@ -257,7 +303,7 @@ class Channel:
         self,
         payload: Any,
 
-        hide_tb: bool = True,
+        hide_tb: bool = False,
 
     ) -> None:
         '''
@@ -434,8 +480,8 @@ class Channel:
         await self.send(aid)
         peer_aid: Aid = await self.recv()
         log.runtime(
-            f'Received hanshake with peer actor,\n'
-            f'{peer_aid}\n'
+            f'Received hanshake with peer\n'
+            f'<= {peer_aid.reprol(sin_uuid=False)}\n'
         )
         # NOTE, we always are referencing the remote peer!
         self.aid = peer_aid

tractor/ipc/_mp_bs.py

@@ -17,29 +17,59 @@
 Utils to tame mp non-SC madeness
 
 '''
+import platform
+
+
 def disable_mantracker():
     '''
-    Disable all ``multiprocessing``` "resource tracking" machinery since
+    Disable all `multiprocessing` "resource tracking" machinery since
     it's an absolute multi-threaded mess of non-SC madness.
 
     '''
-    from multiprocessing import resource_tracker as mantracker
+    from multiprocessing.shared_memory import SharedMemory
 
-    # Tell the "resource tracker" thing to fuck off.
-    class ManTracker(mantracker.ResourceTracker):
-        def register(self, name, rtype):
-            pass
 
-        def unregister(self, name, rtype):
-            pass
+    # 3.13+ only.. can pass `track=False` to disable
+    # all the resource tracker bs.
+    # https://docs.python.org/3/library/multiprocessing.shared_memory.html
+    if (_py_313 := (
+            platform.python_version_tuple()[:-1]
+            >=
+            ('3', '13')
+        )
+    ):
+        from functools import partial
+        return partial(
+            SharedMemory,
+            track=False,
+        )
 
-        def ensure_running(self):
-            pass
+    # !TODO, once we drop 3.12- we can obvi remove all this!
+    else:
+        from multiprocessing import (
+            resource_tracker as mantracker,
+        )
 
-    # "know your land and know your prey"
-    # https://www.dailymotion.com/video/x6ozzco
-    mantracker._resource_tracker = ManTracker()
-    mantracker.register = mantracker._resource_tracker.register
-    mantracker.ensure_running = mantracker._resource_tracker.ensure_running
-    mantracker.unregister = mantracker._resource_tracker.unregister
-    mantracker.getfd = mantracker._resource_tracker.getfd
+        # Tell the "resource tracker" thing to fuck off.
+        class ManTracker(mantracker.ResourceTracker):
+            def register(self, name, rtype):
+                pass
+
+            def unregister(self, name, rtype):
+                pass
+
+            def ensure_running(self):
+                pass
+
+        # "know your land and know your prey"
+        # https://www.dailymotion.com/video/x6ozzco
+        mantracker._resource_tracker = ManTracker()
+        mantracker.register = mantracker._resource_tracker.register
+        mantracker.ensure_running = mantracker._resource_tracker.ensure_running
+        mantracker.unregister = mantracker._resource_tracker.unregister
+        mantracker.getfd = mantracker._resource_tracker.getfd
+
+        # use std type verbatim
+        shmT = SharedMemory
+
+    return shmT

tractor/ipc/_server.py

@@ -26,7 +26,7 @@ from contextlib import (
 from functools import partial
 from itertools import chain
 import inspect
-from pprint import pformat
+import textwrap
 from types import (
     ModuleType,
 )
@@ -43,7 +43,10 @@ from trio import (
     SocketListener,
 )
 
-# from ..devx import _debug
+from ..devx.pformat import (
+    ppfmt,
+    nest_from_op,
+)
 from .._exceptions import (
     TransportClosed,
 )
@@ -141,9 +144,8 @@ async def maybe_wait_on_canced_subs(
 
     ):
         log.cancel(
-            'Waiting on cancel request to peer..\n'
-            f'c)=>\n'
-            f'  |_{chan.aid}\n'
+            'Waiting on cancel request to peer\n'
+            f'c)=> {chan.aid.reprol()}@[{chan.maddr}]\n'
         )
 
         # XXX: this is a soft wait on the channel (and its
@@ -179,7 +181,7 @@ async def maybe_wait_on_canced_subs(
                 log.warning(
                     'Draining msg from disconnected peer\n'
                     f'{chan_info}'
-                    f'{pformat(msg)}\n'
+                    f'{ppfmt(msg)}\n'
                 )
                 # cid: str|None = msg.get('cid')
                 cid: str|None = msg.cid
@@ -248,7 +250,7 @@ async def maybe_wait_on_canced_subs(
                 if children := local_nursery._children:
                     # indent from above local-nurse repr
                     report += (
-                        f'   |_{pformat(children)}\n'
+                        f'   |_{ppfmt(children)}\n'
                     )
 
                 log.warning(report)
@@ -279,8 +281,9 @@ async def maybe_wait_on_canced_subs(
                     log.runtime(
                         f'Peer IPC broke but subproc is alive?\n\n'
 
-                        f'<=x {chan.aid}@{chan.raddr}\n'
-                        f'   |_{proc}\n'
+                        f'<=x {chan.aid.reprol()}@[{chan.maddr}]\n'
+                        f'\n'
+                        f'{proc}\n'
                     )
 
     return local_nursery
@@ -318,15 +321,16 @@ async def handle_stream_from_peer(
     server._no_more_peers = trio.Event()  # unset by making new
 
     # TODO, debug_mode tooling for when hackin this lower layer?
-    # with _debug.maybe_open_crash_handler(
+    # with debug.maybe_open_crash_handler(
     #     pdb=True,
     # ) as boxerr:
 
     chan = Channel.from_stream(stream)
     con_status: str = (
-        'New inbound IPC connection <=\n'
-        f'|_{chan}\n'
+        f'New inbound IPC transport connection\n'
+        f'<=( {stream!r}\n'
     )
+    con_status_steps: str = ''
 
     # initial handshake with peer phase
     try:
@@ -372,7 +376,7 @@ async def handle_stream_from_peer(
     if _pre_chan := server._peers.get(uid):
         familiar: str = 'pre-existing-peer'
     uid_short: str = f'{uid[0]}[{uid[1][-6:]}]'
-    con_status += (
+    con_status_steps += (
         f' -> Handshake with {familiar} `{uid_short}` complete\n'
     )
 
@@ -397,7 +401,7 @@ async def handle_stream_from_peer(
         None,
     )
     if event:
-        con_status += (
+        con_status_steps += (
             ' -> Waking subactor spawn waiters: '
             f'{event.statistics().tasks_waiting}\n'
             f' -> Registered IPC chan for child actor {uid}@{chan.raddr}\n'
@@ -408,7 +412,7 @@ async def handle_stream_from_peer(
         event.set()
 
     else:
-        con_status += (
+        con_status_steps += (
             f' -> Registered IPC chan for peer actor {uid}@{chan.raddr}\n'
         )  # type: ignore
 
@@ -422,8 +426,15 @@ async def handle_stream_from_peer(
     # TODO: can we just use list-ref directly?
     chans.append(chan)
 
-    con_status += ' -> Entering RPC msg loop..\n'
-    log.runtime(con_status)
+    con_status_steps += ' -> Entering RPC msg loop..\n'
+    log.runtime(
+        con_status
+        +
+        textwrap.indent(
+            con_status_steps,
+            prefix=' '*3,  # align to first-ln
+        )
+    )
 
     # Begin channel management - respond to remote requests and
     # process received reponses.
@@ -456,41 +467,67 @@ async def handle_stream_from_peer(
             disconnected=disconnected,
         )
 
-        # ``Channel`` teardown and closure sequence
+        # `Channel` teardown and closure sequence
         # drop ref to channel so it can be gc-ed and disconnected
-        con_teardown_status: str = (
-            f'IPC channel disconnected:\n'
-            f'<=x uid: {chan.aid}\n'
-            f'   |_{pformat(chan)}\n\n'
+        #
+        # -[x]TODO mk this be like
+        # <=x Channel(
+        #     |_field: blah
+        # )>
+        op_repr: str = '<=x '
+        chan_repr: str = nest_from_op(
+            input_op=op_repr,
+            op_suffix='',
+            nest_prefix='',
+            text=chan.pformat(),
+            nest_indent=len(op_repr)-1,
+            rm_from_first_ln='<',
         )
+
+        con_teardown_status: str = (
+            f'IPC channel disconnect\n'
+            f'\n'
+            f'{chan_repr}\n'
+            f'\n'
+        )
+
         chans.remove(chan)
 
         # TODO: do we need to be this pedantic?
         if not chans:
             con_teardown_status += (
-                f'-> No more channels with {chan.aid}'
+                f'-> No more channels with {chan.aid.reprol()!r}\n'
             )
             server._peers.pop(uid, None)
 
-        peers_str: str = ''
-        for uid, chans in server._peers.items():
-            peers_str += (
-                f'uid: {uid}\n'
-            )
-            for i, chan in enumerate(chans):
-                peers_str += (
-                    f' |_[{i}] {pformat(chan)}\n'
+        if peers := list(server._peers.values()):
+            peer_cnt: int = len(peers)
+            if (
+                (first := peers[0][0]) is not chan
+                and
+                not disconnected
+                and
+                peer_cnt > 1
+            ):
+                con_teardown_status += (
+                    f'-> Remaining IPC {peer_cnt-1!r} peers:\n'
                 )
-
-        con_teardown_status += (
-            f'-> Remaining IPC {len(server._peers)} peers: {peers_str}\n'
-        )
+                for chans in server._peers.values():
+                    first: Channel = chans[0]
+                    if not (
+                        first is chan
+                        and
+                        disconnected
+                    ):
+                        con_teardown_status += (
+                            f'  |_{first.aid.reprol()!r} -> {len(chans)!r} chans\n'
+                        )
 
         # No more channels to other actors (at all) registered
         # as connected.
         if not server._peers:
             con_teardown_status += (
-                'Signalling no more peer channel connections'
+                '-> Signalling no more peer connections!\n'
             )
             server._no_more_peers.set()
 
@@ -504,8 +541,8 @@ async def handle_stream_from_peer(
                 and
                 _state.is_debug_mode()
             ):
-                from ..devx import _debug
-                pdb_lock = _debug.Lock
+                from ..devx import debug
+                pdb_lock = debug.Lock
                 pdb_lock._blocked.add(uid)
 
                 # TODO: NEEEDS TO BE TESTED!
@@ -540,7 +577,7 @@ async def handle_stream_from_peer(
                                 f'last disconnected child uid: {uid}\n'
                                 f'locking child uid: {pdb_user_uid}\n'
                             )
-                            await _debug.maybe_wait_for_debugger(
+                            await debug.maybe_wait_for_debugger(
                                 child_in_debug=True
                             )
 
@@ -579,10 +616,10 @@ async def handle_stream_from_peer(
 
 class Endpoint(Struct):
     '''
-    An instance of an IPC "bound" address where the lifetime of the
-    "ability to accept connections" (from clients) and then handle
-    those inbound sessions or sequences-of-packets is determined by
-    a (maybe pair of) nurser(y/ies).
+    An instance of an IPC "bound" address where the lifetime of an
+    "ability to accept connections" and handle the subsequent
+    sequence-of-packets (maybe oriented as sessions) is determined by
+    the underlying nursery scope(s).
 
     '''
     addr: Address
@@ -600,6 +637,24 @@ class Endpoint(Struct):
         MsgTransport,  # handle to encoded-msg transport stream
     ] = {}
 
+    def pformat(
+        self,
+        indent: int = 0,
+        privates: bool = False,
+    ) -> str:
+        type_repr: str = type(self).__name__
+        fmtstr: str = (
+            # !TODO, always be ns aware!
+            # f'|_netns: {netns}\n'
+            f' |.addr: {self.addr!r}\n'
+            f' |_peers: {len(self.peer_tpts)}\n'
+        )
+        return (
+            f'<{type_repr}(\n'
+            f'{fmtstr}'
+            f')>'
+        )
+
     async def start_listener(self) -> SocketListener:
         tpt_mod: ModuleType = inspect.getmodule(self.addr)
         lstnr: SocketListener = await tpt_mod.start_listener(
@@ -639,11 +694,13 @@ class Endpoint(Struct):
 class Server(Struct):
     _parent_tn: Nursery
     _stream_handler_tn: Nursery
+
     # level-triggered sig for whether "no peers are currently
     # connected"; field is **always** set to an instance but
     # initialized with `.is_set() == True`.
     _no_more_peers: trio.Event
 
+    # active eps as allocated by `.listen_on()`
     _endpoints: list[Endpoint] = []
 
     # connection tracking & mgmt
@@ -651,12 +708,19 @@ class Server(Struct):
         str,  # uaid
         list[Channel],  # IPC conns from peer
     ] = defaultdict(list)
+
+    # events-table with entries registered unset while the local
+    # actor is waiting on a new actor to inbound connect, often
+    # a parent waiting on its child just after spawn.
     _peer_connected: dict[
         tuple[str, str],
         trio.Event,
     ] = {}
 
     # syncs for setup/teardown sequences
+    # - null when not yet booted,
+    # - unset when active,
+    # - set when fully shutdown with 0 eps active.
     _shutdown: trio.Event|None = None
 
     # TODO, maybe just make `._endpoints: list[Endpoint]` and
@@ -664,7 +728,6 @@ class Server(Struct):
     # @property
     # def addrs2eps(self) -> dict[Address, Endpoint]:
     #     ...
-
     @property
     def proto_keys(self) -> list[str]:
         return [
@@ -690,7 +753,7 @@ class Server(Struct):
             # TODO: obvi a different server type when we eventually
             # support some others XD
             log.runtime(
-                f'Cancelling server(s) for\n'
+                f'Cancelling server(s) for tpt-protos\n'
                 f'{self.proto_keys!r}\n'
             )
             self._parent_tn.cancel_scope.cancel()
@@ -717,6 +780,14 @@ class Server(Struct):
                 f'protos: {tpt_protos!r}\n'
             )
 
+    def len_peers(
+        self,
+    ) -> int:
+        return len([
+            chan.connected()
+            for chan in chain(*self._peers.values())
+        ])
+
     def has_peers(
         self,
         check_chans: bool = False,
@@ -730,13 +801,11 @@ class Server(Struct):
             has_peers
             and
             check_chans
+            and
+            (peer_cnt := self.len_peers())
         ):
             has_peers: bool = (
-                any(chan.connected()
-                    for chan in chain(
-                        *self._peers.values()
-                    )
-                )
+                peer_cnt > 0
                 and
                 has_peers
             )
@@ -745,10 +814,14 @@ class Server(Struct):
 
     async def wait_for_no_more_peers(
         self,
-        shield: bool = False,
+        # XXX, should this even be allowed?
+        # -> i've seen it cause hangs on teardown
+        #    in `test_resource_cache.py`
+        # _shield: bool = False,
     ) -> None:
-        with trio.CancelScope(shield=shield):
-            await self._no_more_peers.wait()
+        await self._no_more_peers.wait()
+        # with trio.CancelScope(shield=_shield):
+        #     await self._no_more_peers.wait()
 
     async def wait_for_peer(
         self,
@@ -803,30 +876,66 @@ class Server(Struct):
 
         return ev.is_set()
 
-    def pformat(self) -> str:
+    @property
+    def repr_state(self) -> str:
+        '''
+        A `str`-status describing the current state of this
+        IPC server in terms of the current operating "phase".
+
+        '''
+        status = 'server is active'
+        if self.has_peers():
+            peer_cnt: int = self.len_peers()
+            status: str = (
+                f'{peer_cnt!r} peer chans'
+            )
+        else:
+            status: str = 'No peer chans'
+
+        if self.is_shutdown():
+            status: str = 'server-shutdown'
+
+        return status
+
+    def pformat(
+        self,
+        privates: bool = False,
+    ) -> str:
         eps: list[Endpoint] = self._endpoints
 
-        state_repr: str = (
-            f'{len(eps)!r} IPC-endpoints active'
-        )
+        # state_repr: str = (
+        #     f'{len(eps)!r} endpoints active'
+        # )
         fmtstr = (
-            f' |_state: {state_repr}\n'
-            f'   no_more_peers: {self.has_peers()}\n'
+            f' |_state: {self.repr_state!r}\n'
         )
-        if self._shutdown is not None:
-            shutdown_stats: EventStatistics = self._shutdown.statistics()
+        if privates:
+            fmtstr += f'   no_more_peers: {self.has_peers()}\n'
+
+            if self._shutdown is not None:
+                shutdown_stats: EventStatistics = self._shutdown.statistics()
+                fmtstr += (
+                    f'   task_waiting_on_shutdown: {shutdown_stats}\n'
+                )
+
+        if eps := self._endpoints:
+            addrs: list[tuple] = [
+                ep.addr for ep in eps
+            ]
+            repr_eps: str = ppfmt(addrs)
+
             fmtstr += (
-                f'   task_waiting_on_shutdown: {shutdown_stats}\n'
+                f' |_endpoints: {repr_eps}\n'
+                # ^TODO? how to indent closing ']'..
             )
 
-        fmtstr += (
-            # TODO, use the `ppfmt()` helper from `modden`!
-            f' |_endpoints: {pformat(self._endpoints)}\n'
-            f' |_peers: {len(self._peers)} connected\n'
-        )
+        if peers := self._peers:
+            fmtstr += (
+                f' |_peers: {len(peers)} connected\n'
+            )
 
         return (
-            f'<IPCServer(\n'
+            f'<Server(\n'
             f'{fmtstr}'
             f')>\n'
         )
@@ -885,24 +994,34 @@ class Server(Struct):
             )
 
         log.runtime(
-            f'Binding to endpoints for,\n'
-            f'{accept_addrs}\n'
+            f'Binding endpoints\n'
+            f'{ppfmt(accept_addrs)}\n'
         )
         eps: list[Endpoint] = await self._parent_tn.start(
             partial(
                 _serve_ipc_eps,
                 server=self,
-                stream_handler_tn=stream_handler_nursery,
+                stream_handler_tn=(
+                    stream_handler_nursery
+                    or
+                    self._stream_handler_tn
+                ),
                 listen_addrs=accept_addrs,
             )
         )
+        self._endpoints.extend(eps)
+
+        serv_repr: str = nest_from_op(
+            input_op='(>',
+            text=self.pformat(),
+            nest_indent=1,
+        )
         log.runtime(
-            f'Started IPC endpoints\n'
-            f'{eps}\n'
+            f'Started IPC server\n'
+            f'{serv_repr}'
         )
 
-        self._endpoints.extend(eps)
-        # XXX, just a little bit of sanity
+        # XXX, a little sanity on new ep allocations
         group_tn: Nursery|None = None
         ep: Endpoint
         for ep in eps:
@@ -956,9 +1075,13 @@ async def _serve_ipc_eps(
                     stream_handler_tn=stream_handler_tn,
                 )
                 try:
+                    ep_sclang: str = nest_from_op(
+                        input_op='>[',
+                        text=f'{ep.pformat()}',
+                    )
                     log.runtime(
                         f'Starting new endpoint listener\n'
-                        f'{ep}\n'
+                        f'{ep_sclang}\n'
                     )
                     listener: trio.abc.Listener = await ep.start_listener()
                     assert listener is ep._listener
@@ -996,17 +1119,6 @@ async def _serve_ipc_eps(
                     handler_nursery=stream_handler_tn
                 )
             )
-            # TODO, wow make this message better! XD
-            log.runtime(
-                'Started server(s)\n'
-                +
-                '\n'.join([f'|_{addr}' for addr in listen_addrs])
-            )
-
-            log.runtime(
-                f'Started IPC endpoints\n'
-                f'{eps}\n'
-            )
             task_status.started(
                 eps,
             )
@@ -1037,20 +1149,23 @@ async def open_ipc_server(
 
     async with maybe_open_nursery(
         nursery=parent_tn,
-    ) as rent_tn:
+    ) as parent_tn:
         no_more_peers = trio.Event()
         no_more_peers.set()
 
         ipc_server = IPCServer(
-            _parent_tn=rent_tn,
-            _stream_handler_tn=stream_handler_tn or rent_tn,
+            _parent_tn=parent_tn,
+            _stream_handler_tn=(
+                stream_handler_tn
+                or
+                parent_tn
+            ),
             _no_more_peers=no_more_peers,
         )
         try:
             yield ipc_server
             log.runtime(
-                f'Waiting on server to shutdown or be cancelled..\n'
-                f'{ipc_server}'
+                'Server-tn running until terminated\n'
             )
             # TODO? when if ever would we want/need this?
             # with trio.CancelScope(shield=True):

tractor/ipc/_shm.py

@@ -23,14 +23,15 @@ considered optional within the context of this runtime-library.
 
 """
 from __future__ import annotations
+from multiprocessing import shared_memory as shm
+from multiprocessing.shared_memory import (
+    # SharedMemory,
+    ShareableList,
+)
+import platform
 from sys import byteorder
 import time
 from typing import Optional
-from multiprocessing import shared_memory as shm
-from multiprocessing.shared_memory import (
-    SharedMemory,
-    ShareableList,
-)
 
 from msgspec import (
     Struct,
@@ -61,7 +62,7 @@ except ImportError:
 log = get_logger(__name__)
 
 
-disable_mantracker()
+SharedMemory = disable_mantracker()
 
 
 class SharedInt:
@@ -789,11 +790,23 @@ def open_shm_list(
         readonly=readonly,
     )
 
+    # TODO, factor into a @actor_fixture acm-API?
+    # -[ ] also `@maybe_actor_fixture()` which inludes
+    #     the .current_actor() convenience check?
+    #   |_ orr can that just be in the sin-maybe-version?
+    #
     # "close" attached shm on actor teardown
     try:
         actor = tractor.current_actor()
+
         actor.lifetime_stack.callback(shml.shm.close)
-        actor.lifetime_stack.callback(shml.shm.unlink)
+
+        # XXX on 3.13+ we don't need to call this?
+        # -> bc we pass `track=False` for `SharedMemeory` orr?
+        if (
+            platform.python_version_tuple()[:-1] < ('3', '13')
+        ):
+            actor.lifetime_stack.callback(shml.shm.unlink)
     except RuntimeError:
         log.warning('tractor runtime not active, skipping teardown steps')
 

tractor/ipc/_tcp.py

@@ -160,10 +160,9 @@ async def start_listener(
     Start a TCP socket listener on the given `TCPAddress`.
 
     '''
-    log.info(
-        f'Attempting to bind TCP socket\n'
-        f'>[\n'
-        f'|_{addr}\n'
+    log.runtime(
+        f'Trying socket bind\n'
+        f'>[ {addr}\n'
     )
     # ?TODO, maybe we should just change the lower-level call this is
     # using internall per-listener?
@@ -178,11 +177,10 @@ async def start_listener(
     assert len(listeners) == 1
     listener = listeners[0]
     host, port = listener.socket.getsockname()[:2]
-
+    bound_addr: TCPAddress = type(addr).from_addr((host, port))
     log.info(
         f'Listening on TCP socket\n'
-        f'[>\n'
-        f' |_{addr}\n'
+        f'[> {bound_addr}\n'
     )
     return listener
 

tractor/ipc/_transport.py

@@ -430,20 +430,25 @@ class MsgpackTransport(MsgTransport):
                 return await self.stream.send_all(size + bytes_data)
             except (
                 trio.BrokenResourceError,
-            ) as bre:
-                trans_err = bre
+                trio.ClosedResourceError,
+            ) as _re:
+                trans_err = _re
                 tpt_name: str = f'{type(self).__name__!r}'
+
                 match trans_err:
+
+                    # XXX, specifc to UDS transport and its,
+                    # well, "speediness".. XD
+                    # |_ likely todo with races related to how fast
+                    #    the socket is setup/torn-down on linux
+                    #    as it pertains to rando pings from the
+                    #    `.discovery` subsys and protos.
                     case trio.BrokenResourceError() if (
-                        '[Errno 32] Broken pipe' in trans_err.args[0]
-                        # ^XXX, specifc to UDS transport and its,
-                        # well, "speediness".. XD
-                        # |_ likely todo with races related to how fast
-                        #    the socket is setup/torn-down on linux
-                        #    as it pertains to rando pings from the
-                        #    `.discovery` subsys and protos.
+                        '[Errno 32] Broken pipe'
+                        in
+                        trans_err.args[0]
                     ):
-                        raise TransportClosed.from_src_exc(
+                        tpt_closed = TransportClosed.from_src_exc(
                             message=(
                                 f'{tpt_name} already closed by peer\n'
                             ),
@@ -451,14 +456,31 @@ class MsgpackTransport(MsgTransport):
                             src_exc=trans_err,
                             raise_on_report=True,
                             loglevel='transport',
-                        ) from bre
+                        )
+                        raise tpt_closed from trans_err
+
+                    # case trio.ClosedResourceError() if (
+                    #     'this socket was already closed'
+                    #     in
+                    #     trans_err.args[0]
+                    # ):
+                    #     tpt_closed = TransportClosed.from_src_exc(
+                    #         message=(
+                    #             f'{tpt_name} already closed by peer\n'
+                    #         ),
+                    #         body=f'{self}\n',
+                    #         src_exc=trans_err,
+                    #         raise_on_report=True,
+                    #         loglevel='transport',
+                    #     )
+                    #     raise tpt_closed from trans_err
 
                     # unless the disconnect condition falls under "a
                     # normal operation breakage" we usualy console warn
                     # about it.
                     case _:
                         log.exception(
-                            '{tpt_name} layer failed pre-send ??\n'
+                            f'{tpt_name} layer failed pre-send ??\n'
                         )
                         raise trans_err
 
@@ -503,7 +525,7 @@ class MsgpackTransport(MsgTransport):
     def pformat(self) -> str:
         return (
             f'<{type(self).__name__}(\n'
-            f' |_peers: 2\n'
+            f' |_peers: 1\n'
             f'   laddr: {self._laddr}\n'
             f'   raddr: {self._raddr}\n'
             # f'\n'

tractor/ipc/_uds.py

@@ -18,6 +18,9 @@ Unix Domain Socket implementation of tractor.ipc._transport.MsgTransport protoco
 
 '''
 from __future__ import annotations
+from contextlib import (
+    contextmanager as cm,
+)
 from pathlib import Path
 import os
 from socket import (
@@ -29,6 +32,7 @@ from socket import (
 )
 import struct
 from typing import (
+    Type,
     TYPE_CHECKING,
     ClassVar,
 )
@@ -99,8 +103,6 @@ class UDSAddress(
             self.filedir
             or
             self.def_bindspace
-            # or
-            # get_rt_dir()
         )
 
     @property
@@ -205,12 +207,35 @@ class UDSAddress(
             f']'
         )
 
+@cm
+def _reraise_as_connerr(
+    src_excs: tuple[Type[Exception]],
+    addr: UDSAddress,
+):
+    try:
+        yield
+    except src_excs as src_exc:
+        raise ConnectionError(
+            f'Bad UDS socket-filepath-as-address ??\n'
+            f'{addr}\n'
+            f' |_sockpath: {addr.sockpath}\n'
+            f'\n'
+            f'from src: {src_exc!r}\n'
+        ) from src_exc
+
 
 async def start_listener(
     addr: UDSAddress,
     **kwargs,
 ) -> SocketListener:
-    # sock = addr._sock = socket.socket(
+    '''
+    Start listening for inbound connections via
+    a `trio.SocketListener` (task) which `socket.bind()`s on `addr`.
+
+    Note, if the `UDSAddress.bindspace: Path` directory dne it is
+    implicitly created.
+
+    '''
     sock = socket.socket(
         socket.AF_UNIX,
         socket.SOCK_STREAM
@@ -221,17 +246,25 @@ async def start_listener(
         f'|_{addr}\n'
     )
 
+    # ?TODO? should we use the `actor.lifetime_stack`
+    # to rm on shutdown?
     bindpath: Path = addr.sockpath
-    try:
+    if not (bs := addr.bindspace).is_dir():
+        log.info(
+            'Creating bindspace dir in file-sys\n'
+            f'>{{\n'
+            f'|_{bs!r}\n'
+        )
+        bs.mkdir()
+
+    with _reraise_as_connerr(
+        src_excs=(
+            FileNotFoundError,
+            OSError,
+        ),
+        addr=addr
+    ):
         await sock.bind(str(bindpath))
-    except (
-        FileNotFoundError,
-    ) as fdne:
-        raise ConnectionError(
-            f'Bad UDS socket-filepath-as-address ??\n'
-            f'{addr}\n'
-            f' |_sockpath: {addr.sockpath}\n'
-        ) from fdne
 
     sock.listen(1)
     log.info(
@@ -356,27 +389,30 @@ class MsgpackUDSStream(MsgpackTransport):
         # `.setsockopt()` call tells the OS provide it; the client
         # pid can then be read on server/listen() side via
         # `get_peer_info()` above.
-        try:
+
+        with _reraise_as_connerr(
+            src_excs=(
+                FileNotFoundError,
+            ),
+            addr=addr
+        ):
             stream = await open_unix_socket_w_passcred(
                 str(sockpath),
                 **kwargs
             )
-        except (
-            FileNotFoundError,
-        ) as fdne:
-            raise ConnectionError(
-                f'Bad UDS socket-filepath-as-address ??\n'
-                f'{addr}\n'
-                f' |_sockpath: {sockpath}\n'
-            ) from fdne
 
-        stream = MsgpackUDSStream(
+        tpt_stream = MsgpackUDSStream(
             stream,
             prefix_size=prefix_size,
             codec=codec
         )
-        stream._raddr = addr
-        return stream
+        # XXX assign from new addrs after peer-PID extract!
+        (
+            tpt_stream._laddr,
+            tpt_stream._raddr,
+        ) = cls.get_stream_addrs(stream)
+
+        return tpt_stream
 
     @classmethod
     def get_stream_addrs(

tractor/log.py

@@ -81,10 +81,35 @@ BOLD_PALETTE = {
 }
 
 
+def at_least_level(
+    log: Logger|LoggerAdapter,
+    level: int|str,
+) -> bool:
+    '''
+    Predicate to test if a given level is active.
+
+    '''
+    if isinstance(level, str):
+        level: int = CUSTOM_LEVELS[level.upper()]
+
+    if log.getEffectiveLevel() <= level:
+        return True
+    return False
+
+
 # TODO: this isn't showing the correct '{filename}'
 # as it did before..
 class StackLevelAdapter(LoggerAdapter):
 
+    def at_least_level(
+        self,
+        level: str,
+    ) -> bool:
+        return at_least_level(
+            log=self,
+            level=level,
+        )
+
     def transport(
         self,
         msg: str,
@@ -270,7 +295,9 @@ def get_logger(
     subsys_spec: str|None = None,
 
 ) -> StackLevelAdapter:
-    '''Return the package log or a sub-logger for ``name`` if provided.
+    '''
+    Return the `tractor`-library root logger or a sub-logger for
+    `name` if provided.
 
     '''
     log: Logger
@@ -282,7 +309,7 @@ def get_logger(
         name != _proj_name
     ):
 
-        # NOTE: for handling for modules that use ``get_logger(__name__)``
+        # NOTE: for handling for modules that use `get_logger(__name__)`
         # we make the following stylistic choice:
         # - always avoid duplicate project-package token
         #   in msg output: i.e. tractor.tractor.ipc._chan.py in header
@@ -331,7 +358,7 @@ def get_logger(
 
 def get_console_log(
     level: str|None = None,
-    logger: Logger|None = None,
+    logger: Logger|StackLevelAdapter|None = None,
     **kwargs,
 
 ) -> LoggerAdapter:
@@ -344,12 +371,23 @@ def get_console_log(
     Yeah yeah, i know we can use `logging.config.dictConfig()`. You do it.
 
     '''
-    log = get_logger(
-        logger=logger,
-        **kwargs
-    )  # set a root logger
-    logger: Logger = log.logger
+    # get/create a stack-aware-adapter
+    if (
+        logger
+        and
+        isinstance(logger, StackLevelAdapter)
+    ):
+        # XXX, for ex. when passed in by a caller wrapping some
+        # other lib's logger instance with our level-adapter.
+        log = logger
 
+    else:
+        log: StackLevelAdapter = get_logger(
+            logger=logger,
+            **kwargs
+        )
+
+    logger: Logger|StackLevelAdapter = log.logger
     if not level:
         return log
 
@@ -367,10 +405,7 @@ def get_console_log(
             None,
         )
     ):
-        fmt = LOG_FORMAT
-        # if logger:
-        #     fmt = None
-
+        fmt: str = LOG_FORMAT  # always apply our format?
         handler = StreamHandler()
         formatter = colorlog.ColoredFormatter(
             fmt=fmt,
@@ -391,19 +426,3 @@ def get_loglevel() -> str:
 
 # global module logger for tractor itself
 log: StackLevelAdapter = get_logger('tractor')
-
-
-def at_least_level(
-    log: Logger|LoggerAdapter,
-    level: int|str,
-) -> bool:
-    '''
-    Predicate to test if a given level is active.
-
-    '''
-    if isinstance(level, str):
-        level: int = CUSTOM_LEVELS[level.upper()]
-
-    if log.getEffectiveLevel() <= level:
-        return True
-    return False

tractor/msg/_ops.py

@@ -210,12 +210,14 @@ class PldRx(Struct):
         match msg:
             case Return()|Error():
                 log.runtime(
-                    f'Rxed final outcome msg\n'
+                    f'Rxed final-outcome msg\n'
+                    f'\n'
                     f'{msg}\n'
                 )
             case Stop():
                 log.runtime(
                     f'Rxed stream stopped msg\n'
+                    f'\n'
                     f'{msg}\n'
                 )
                 if passthrough_non_pld_msgs:
@@ -261,8 +263,9 @@ class PldRx(Struct):
         if (
             type(msg) is Return
         ):
-            log.info(
+            log.runtime(
                 f'Rxed final result msg\n'
+                f'\n'
                 f'{msg}\n'
             )
         return self.decode_pld(
@@ -304,10 +307,13 @@ class PldRx(Struct):
                 try:
                     pld: PayloadT = self._pld_dec.decode(pld)
                     log.runtime(
-                        'Decoded msg payload\n\n'
+                        f'Decoded payload for\n'
+                        # f'\n'
                         f'{msg}\n'
-                        f'where payload decoded as\n'
-                        f'|_pld={pld!r}\n'
+                        # ^TODO?, ideally just render with `,
+                        # pld={decode}` in the `msg.pformat()`??
+                        f'where, '
+                        f'{type(msg).__name__}.pld={pld!r}\n'
                     )
                     return pld
                 except TypeError as typerr:
@@ -494,7 +500,8 @@ def limit_plds(
 
     finally:
         log.runtime(
-            'Reverted to previous payload-decoder\n\n'
+            f'Reverted to previous payload-decoder\n'
+            f'\n'
             f'{orig_pldec}\n'
         )
         # sanity on orig settings
@@ -606,10 +613,9 @@ async def drain_to_final_msg(
             #       msg: dict = await ctx._rx_chan.receive()
             #   if res_cs.cancelled_caught:
             #
-            # -[ ] make sure pause points work here for REPLing
+            # -[x] make sure pause points work here for REPLing
             #   the runtime itself; i.e. ensure there's no hangs!
-            # |_from tractor.devx._debug import pause
-            #   await pause()
+            #  |_see masked code below in .cancel_called path
 
         # NOTE: we get here if the far end was
         # `ContextCancelled` in 2 cases:
@@ -629,7 +635,8 @@ async def drain_to_final_msg(
                     (local_cs := rent_n.cancel_scope).cancel_called
                 ):
                     log.cancel(
-                        'RPC-ctx cancelled by local-parent scope during drain!\n\n'
+                        f'RPC-ctx cancelled by local-parent scope during drain!\n'
+                        f'\n'
                         f'c}}>\n'
                         f' |_{rent_n}\n'
                         f'   |_.cancel_scope = {local_cs}\n'
@@ -644,6 +651,10 @@ async def drain_to_final_msg(
                     f'IPC ctx cancelled externally during result drain ?\n'
                     f'{ctx}'
                 )
+                # XXX, for tracing `Cancelled`..
+                # from tractor.devx.debug import pause
+                # await pause(shield=True)
+
             # CASE 2: mask the local cancelled-error(s)
             # only when we are sure the remote error is
             # the source cause of this local task's
@@ -663,7 +674,8 @@ async def drain_to_final_msg(
             # final result arrived!
             case Return():
                 log.runtime(
-                    'Context delivered final draining msg:\n'
+                    f'Context delivered final draining msg\n'
+                    f'\n'
                     f'{pretty_struct.pformat(msg)}'
                 )
                 ctx._result: Any = pld
@@ -697,12 +709,14 @@ async def drain_to_final_msg(
                 ):
                     log.cancel(
                         'Cancelling `MsgStream` drain since '
-                        f'{reason}\n\n'
+                        f'{reason}\n'
+                        f'\n'
                         f'<= {ctx.chan.uid}\n'
-                        f'  |_{ctx._nsf}()\n\n'
+                        f'  |_{ctx._nsf}()\n'
+                        f'\n'
                         f'=> {ctx._task}\n'
-                        f'  |_{ctx._stream}\n\n'
-
+                        f'  |_{ctx._stream}\n'
+                        f'\n'
                         f'{pretty_struct.pformat(msg)}\n'
                     )
                     break
@@ -739,7 +753,8 @@ async def drain_to_final_msg(
             case Stop():
                 pre_result_drained.append(msg)
                 log.runtime(  # normal/expected shutdown transaction
-                    'Remote stream terminated due to "stop" msg:\n\n'
+                    f'Remote stream terminated due to "stop" msg\n'
+                    f'\n'
                     f'{pretty_struct.pformat(msg)}\n'
                 )
                 continue
@@ -814,7 +829,8 @@ async def drain_to_final_msg(
 
     else:
         log.cancel(
-            'Skipping `MsgStream` drain since final outcome is set\n\n'
+            f'Skipping `MsgStream` drain since final outcome is set\n'
+            f'\n'
             f'{ctx.outcome}\n'
         )
 

tractor/msg/pretty_struct.py

@@ -20,6 +20,7 @@ Prettified version of `msgspec.Struct` for easier console grokin.
 '''
 from __future__ import annotations
 from collections import UserList
+import textwrap
 from typing import (
     Any,
     Iterator,
@@ -105,27 +106,11 @@ def iter_fields(struct: Struct) -> Iterator[
         )
 
 
-def pformat(
+def iter_struct_ppfmt_lines(
     struct: Struct,
-    field_indent: int = 2,
-    indent: int = 0,
+    field_indent: int = 0,
+) -> Iterator[tuple[str, str]]:
 
-) -> str:
-    '''
-    Recursion-safe `pprint.pformat()` style formatting of
-    a `msgspec.Struct` for sane reading by a human using a REPL.
-
-    '''
-    # global whitespace indent
-    ws: str = ' '*indent
-
-    # field whitespace indent
-    field_ws: str = ' '*(field_indent + indent)
-
-    # qtn: str = ws + struct.__class__.__qualname__
-    qtn: str = struct.__class__.__qualname__
-
-    obj_str: str = ''  # accumulator
     fi: structs.FieldInfo
     k: str
     v: Any
@@ -135,15 +120,18 @@ def pformat(
         # ..]` over .__name__ == `Literal` but still get only the
         # latter for simple types like `str | int | None` etc..?
         ft: type = fi.type
-        typ_name: str = getattr(ft, '__name__', str(ft))
+        typ_name: str = getattr(
+            ft,
+            '__name__',
+            str(ft)
+        ).replace(' ', '')
 
         # recurse to get sub-struct's `.pformat()` output Bo
         if isinstance(v, Struct):
-            val_str: str =  v.pformat(
-                indent=field_indent + indent,
-                field_indent=indent + field_indent,
+            yield from iter_struct_ppfmt_lines(
+                struct=v,
+                field_indent=field_indent+field_indent,
             )
-
         else:
             val_str: str = repr(v)
 
@@ -161,8 +149,39 @@ def pformat(
                 # raise
                 # return _Struct.__repr__(struct)
 
-        # TODO: LOLOL use `textwrap.indent()` instead dawwwwwg!
-        obj_str += (field_ws + f'{k}: {typ_name} = {val_str},\n')
+        yield (
+            ' '*field_indent,  # indented ws prefix
+            f'{k}: {typ_name} = {val_str},',  # field's repr line content
+        )
+
+
+def pformat(
+    struct: Struct,
+    field_indent: int = 2,
+    indent: int = 0,
+) -> str:
+    '''
+    Recursion-safe `pprint.pformat()` style formatting of
+    a `msgspec.Struct` for sane reading by a human using a REPL.
+
+    '''
+    obj_str: str = ''  # accumulator
+    for prefix, field_repr, in iter_struct_ppfmt_lines(
+        struct,
+        field_indent=field_indent,
+    ):
+        obj_str += f'{prefix}{field_repr}\n'
+
+    # global whitespace indent
+    ws: str = ' '*indent
+    if indent:
+        obj_str: str = textwrap.indent(
+            text=obj_str,
+            prefix=ws,
+        )
+
+    # qtn: str = ws + struct.__class__.__qualname__
+    qtn: str = struct.__class__.__qualname__
 
     return (
         f'{qtn}(\n'

tractor/msg/types.py

@@ -154,6 +154,39 @@ class Aid(
     #     should also include at least `.pid` (equiv to port for tcp)
     #     and/or host-part always?
 
+    @property
+    def uid(self) -> tuple[str, str]:
+        '''
+        Legacy actor "unique-id" pair format.
+
+        '''
+        return (
+            self.name,
+            self.uuid,
+        )
+
+    def reprol(
+        self,
+        sin_uuid: bool = True,
+    ) -> str:
+        if not sin_uuid:
+            return (
+                f'{self.name}[{self.uuid[:6]}]@{self.pid!r}'
+            )
+        return (
+            f'{self.name}@{self.pid!r}'
+        )
+
+    # mk hashable via `.uuid`
+    def __hash__(self) -> int:
+        return hash(self.uuid)
+
+    def __eq__(self, other: Aid) -> bool:
+        return self.uuid == other.uuid
+
+    # use pretty fmt since often repr-ed for console/log
+    __repr__ = pretty_struct.Struct.__repr__
+
 
 class SpawnSpec(
     pretty_struct.Struct,

tractor/to_asyncio.py

@@ -38,7 +38,6 @@ from typing import (
 import tractor
 from tractor._exceptions import (
     InternalError,
-    is_multi_cancelled,
     TrioTaskExited,
     TrioCancelled,
     AsyncioTaskExited,
@@ -49,7 +48,7 @@ from tractor._state import (
     _runtime_vars,
 )
 from tractor._context import Unresolved
-from tractor.devx import _debug
+from tractor.devx import debug
 from tractor.log import (
     get_logger,
     StackLevelAdapter,
@@ -59,6 +58,9 @@ from tractor.log import (
 # from tractor.msg import (
 #     pretty_struct,
 # )
+from tractor.trionics import (
+    is_multi_cancelled,
+)
 from tractor.trionics._broadcast import (
     broadcast_receiver,
     BroadcastReceiver,
@@ -128,6 +130,7 @@ class LinkedTaskChannel(
     _trio_task: trio.Task
     _aio_task_complete: trio.Event
 
+    _closed_by_aio_task: bool = False
     _suppress_graceful_exits: bool = True
 
     _trio_err: BaseException|None = None
@@ -206,10 +209,15 @@ class LinkedTaskChannel(
     async def aclose(self) -> None:
         await self._from_aio.aclose()
 
-    def started(
+    # ?TODO? async version of this?
+    def started_nowait(
         self,
         val: Any = None,
     ) -> None:
+        '''
+        Synchronize aio-side with its trio-parent.
+
+        '''
         self._aio_started_val = val
         return self._to_trio.send_nowait(val)
 
@@ -240,6 +248,7 @@ class LinkedTaskChannel(
             # cycle on the trio side?
             # await trio.lowlevel.checkpoint()
             return await self._from_aio.receive()
+
         except BaseException as err:
             async with translate_aio_errors(
                 chan=self,
@@ -317,7 +326,7 @@ def _run_asyncio_task(
     qsize: int = 1,
     provide_channels: bool = False,
     suppress_graceful_exits: bool = True,
-    hide_tb: bool = False,
+    hide_tb: bool = True,
     **kwargs,
 
 ) -> LinkedTaskChannel:
@@ -345,18 +354,6 @@ def _run_asyncio_task(
         # value otherwise it would just return ;P
         assert qsize > 1
 
-    if provide_channels:
-        assert 'to_trio' in args
-
-    # allow target func to accept/stream results manually by name
-    if 'to_trio' in args:
-        kwargs['to_trio'] = to_trio
-
-    if 'from_trio' in args:
-        kwargs['from_trio'] = from_trio
-
-    coro = func(**kwargs)
-
     trio_task: trio.Task = trio.lowlevel.current_task()
     trio_cs = trio.CancelScope()
     aio_task_complete = trio.Event()
@@ -371,6 +368,25 @@ def _run_asyncio_task(
         _suppress_graceful_exits=suppress_graceful_exits,
     )
 
+    # allow target func to accept/stream results manually by name
+    if 'to_trio' in args:
+        kwargs['to_trio'] = to_trio
+
+    if 'from_trio' in args:
+        kwargs['from_trio'] = from_trio
+
+    if 'chan' in args:
+        kwargs['chan'] = chan
+
+    if provide_channels:
+        assert (
+            'to_trio' in args
+            or
+            'chan' in args
+        )
+
+    coro = func(**kwargs)
+
     async def wait_on_coro_final_result(
         to_trio: trio.MemorySendChannel,
         coro: Awaitable,
@@ -443,9 +459,23 @@ def _run_asyncio_task(
                         f'Task exited with final result: {result!r}\n'
                     )
 
-                # only close the sender side which will relay
-                # a `trio.EndOfChannel` to the trio (consumer) side.
+                # XXX ALWAYS close the child-`asyncio`-task-side's
+                # `to_trio` handle which will in turn relay
+                # a `trio.EndOfChannel` to the `trio`-parent.
+                # Consequently the parent `trio` task MUST ALWAYS
+                # check for any `chan._aio_err` to be raised when it
+                # receives an EoC.
+                #
+                # NOTE, there are 2 EoC cases,
+                # - normal/graceful EoC due to the aio-side actually
+                #   terminating its "streaming", but the task did not
+                #   error and is not yet complete.
+                #
+                # - the aio-task terminated and we specially mark the
+                #   closure as due to the `asyncio.Task`'s exit.
+                #
                 to_trio.close()
+                chan._closed_by_aio_task = True
 
             aio_task_complete.set()
             log.runtime(
@@ -479,12 +509,12 @@ def _run_asyncio_task(
     if (
         debug_mode()
         and
-        (greenback := _debug.maybe_import_greenback(
+        (greenback := debug.maybe_import_greenback(
             force_reload=True,
             raise_not_found=False,
         ))
     ):
-        log.info(
+        log.devx(
             f'Bestowing `greenback` portal for `asyncio`-task\n'
             f'{task}\n'
         )
@@ -643,8 +673,9 @@ def _run_asyncio_task(
                 not trio_cs.cancel_called
             ):
                 log.cancel(
-                    f'Cancelling `trio` side due to aio-side src exc\n'
-                    f'{curr_aio_err}\n'
+                    f'Cancelling trio-side due to aio-side src exc\n'
+                    f'\n'
+                    f'{curr_aio_err!r}\n'
                     f'\n'
                     f'(c>\n'
                     f'  |_{trio_task}\n'
@@ -756,6 +787,7 @@ async def translate_aio_errors(
     aio_done_before_trio: bool = aio_task.done()
     assert aio_task
     trio_err: BaseException|None = None
+    eoc: trio.EndOfChannel|None = None
     try:
         yield  # back to one of the cross-loop apis
     except trio.Cancelled as taskc:
@@ -787,12 +819,48 @@ async def translate_aio_errors(
         # )
         # raise
 
-    # XXX always passthrough EoC since this translator is often
-    # called from `LinkedTaskChannel.receive()` which we want
-    # passthrough and further we have no special meaning for it in
-    # terms of relaying errors or signals from the aio side!
-    except trio.EndOfChannel as eoc:
-        trio_err = chan._trio_err = eoc
+    # XXX EoC is a special SIGNAL from the aio-side here!
+    # There are 2 cases to handle:
+    # 1. the "EoC passthrough" case.
+    #   - the aio-task actually closed the channel "gracefully" and
+    #     the trio-task should unwind any ongoing channel
+    #     iteration/receiving,
+    #  |_this exc-translator wraps calls to `LinkedTaskChannel.receive()`
+    #    in which case we want to relay the actual "end-of-chan" for
+    #    iteration purposes.
+    #
+    # 2. relaying the "asyncio.Task termination" case.
+    #   - if the aio-task terminates, maybe with an error, AND the
+    #    `open_channel_from()` API was used, it will always signal
+    #    that termination.
+    #  |_`wait_on_coro_final_result()` always calls
+    #    `to_trio.close()` when `provide_channels=True` so we need to
+    #    always check if there is an aio-side exc which needs to be
+    #    relayed to the parent trio side!
+    #  |_in this case the special `chan._closed_by_aio_task` is
+    #    ALWAYS set.
+    #
+    except trio.EndOfChannel as _eoc:
+        eoc = _eoc
+        if (
+            chan._closed_by_aio_task
+            and
+            aio_err
+        ):
+            log.cancel(
+                f'The asyncio-child task terminated due to error\n'
+                f'{aio_err!r}\n'
+            )
+            chan._trio_to_raise = aio_err
+            trio_err = chan._trio_err = eoc
+            #
+            # ?TODO?, raise something like a,
+            # chan._trio_to_raise = AsyncioErrored()
+            # BUT, with the tb rewritten to reflect the underlying
+            # call stack?
+        else:
+            trio_err = chan._trio_err = eoc
+
         raise eoc
 
     # NOTE ALSO SEE the matching note in the `cancel_trio()` asyncio
@@ -841,7 +909,7 @@ async def translate_aio_errors(
     except BaseException as _trio_err:
         trio_err = chan._trio_err = _trio_err
         # await tractor.pause(shield=True)  # workx!
-        entered: bool = await _debug._maybe_enter_pm(
+        entered: bool = await debug._maybe_enter_pm(
             trio_err,
             api_frame=inspect.currentframe(),
         )
@@ -1045,7 +1113,7 @@ async def translate_aio_errors(
         #
         if wait_on_aio_task:
             await chan._aio_task_complete.wait()
-            log.info(
+            log.debug(
                 'asyncio-task is done and unblocked trio-side!\n'
             )
 
@@ -1062,11 +1130,17 @@ async def translate_aio_errors(
         trio_to_raise: (
             AsyncioCancelled|
             AsyncioTaskExited|
+            Exception|  # relayed from aio-task
             None
         ) = chan._trio_to_raise
 
+        raise_from: Exception = (
+            trio_err if (aio_err is trio_to_raise)
+            else aio_err
+        )
+
         if not suppress_graceful_exits:
-            raise trio_to_raise from (aio_err or trio_err)
+            raise trio_to_raise from raise_from
 
         if trio_to_raise:
             match (
@@ -1099,7 +1173,7 @@ async def translate_aio_errors(
                         )
                         return
                 case _:
-                    raise trio_to_raise from (aio_err or trio_err)
+                    raise trio_to_raise from raise_from
 
         # Check if the asyncio-side is the cause of the trio-side
         # error.
@@ -1165,7 +1239,6 @@ async def run_task(
 
 @acm
 async def open_channel_from(
-
     target: Callable[..., Any],
     suppress_graceful_exits: bool = True,
     **target_kwargs,
@@ -1199,7 +1272,6 @@ async def open_channel_from(
                     # deliver stream handle upward
                     yield first, chan
             except trio.Cancelled as taskc:
-                # await tractor.pause(shield=True)  # ya it worx ;)
                 if cs.cancel_called:
                     if isinstance(chan._trio_to_raise, AsyncioCancelled):
                         log.cancel(
@@ -1406,7 +1478,7 @@ def run_as_asyncio_guest(
             )
             # XXX make it obvi we know this isn't supported yet!
             assert 0
-            # await _debug.maybe_init_greenback(
+            # await debug.maybe_init_greenback(
             #     force_reload=True,
             # )
 

tractor/trionics/__init__.py

@@ -31,4 +31,9 @@ from ._broadcast import (
 )
 from ._beg import (
     collapse_eg as collapse_eg,
+    get_collapsed_eg as get_collapsed_eg,
+    is_multi_cancelled as is_multi_cancelled,
+)
+from ._taskc import (
+    maybe_raise_from_masking_exc as maybe_raise_from_masking_exc,
 )

tractor/trionics/_beg.py

@@ -15,31 +15,94 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 '''
-`BaseExceptionGroup` related utils and helpers pertaining to
-first-class-`trio` from a historical perspective B)
+`BaseExceptionGroup` utils and helpers pertaining to
+first-class-`trio` from a "historical" perspective, like "loose
+exception group" task-nurseries.
 
 '''
 from contextlib import (
     asynccontextmanager as acm,
 )
+from typing import (
+    Literal,
+    Type,
+)
+
+import trio
+# from trio._core._concat_tb import (
+#     concat_tb,
+# )
 
 
-def maybe_collapse_eg(
-    beg: BaseExceptionGroup,
+# XXX NOTE
+# taken verbatim from `trio._core._run` except,
+# - remove the NONSTRICT_EXCEPTIONGROUP_NOTE deprecation-note
+#   guard-check; we know we want an explicit collapse.
+# - mask out tb rewriting in collapse case, i don't think it really
+#   matters?
+#
+def collapse_exception_group(
+    excgroup: BaseExceptionGroup[BaseException],
 ) -> BaseException:
+    """Recursively collapse any single-exception groups into that single contained
+    exception.
+
+    """
+    exceptions = list(excgroup.exceptions)
+    modified = False
+    for i, exc in enumerate(exceptions):
+        if isinstance(exc, BaseExceptionGroup):
+            new_exc = collapse_exception_group(exc)
+            if new_exc is not exc:
+                modified = True
+                exceptions[i] = new_exc
+
+    if (
+        len(exceptions) == 1
+        and isinstance(excgroup, BaseExceptionGroup)
+
+        # XXX trio's loose-setting condition..
+        # and NONSTRICT_EXCEPTIONGROUP_NOTE in getattr(excgroup, "__notes__", ())
+    ):
+        # exceptions[0].__traceback__ = concat_tb(
+        #     excgroup.__traceback__,
+        #     exceptions[0].__traceback__,
+        # )
+        return exceptions[0]
+    elif modified:
+        return excgroup.derive(exceptions)
+    else:
+        return excgroup
+
+
+def get_collapsed_eg(
+    beg: BaseExceptionGroup,
+
+) -> BaseException|None:
     '''
-    If the input beg can collapse to a single non-eg sub-exception,
-    return it instead.
+    If the input beg can collapse to a single sub-exception which is
+    itself **not** an eg, return it.
 
     '''
-    if len(excs := beg.exceptions) == 1:
-        return excs[0]
+    maybe_exc = collapse_exception_group(beg)
+    if maybe_exc is beg:
+        return None
 
-    return beg
+    return maybe_exc
 
 
 @acm
-async def collapse_eg():
+async def collapse_eg(
+    hide_tb: bool = True,
+
+    # XXX, for ex. will always show begs containing single taskc
+    ignore: set[Type[BaseException]] = {
+        # trio.Cancelled,
+    },
+    add_notes: bool = True,
+
+    bp: bool = False,
+):
     '''
     If `BaseExceptionGroup` raised in the body scope is
     "collapse-able" (in the same way that
@@ -47,12 +110,114 @@ async def collapse_eg():
     only raise the lone emedded non-eg in in place.
 
     '''
+    __tracebackhide__: bool = hide_tb
     try:
         yield
-    except* BaseException as beg:
-        if (
-            exc := maybe_collapse_eg(beg)
-        ) is not beg:
-            raise exc
+    except BaseExceptionGroup as _beg:
+        beg = _beg
 
-        raise beg
+        if (
+            bp
+            and
+            len(beg.exceptions) > 1
+        ):
+            import tractor
+            if tractor.current_actor(
+                err_on_no_runtime=False,
+            ):
+                await tractor.pause(shield=True)
+            else:
+                breakpoint()
+
+        if (
+            (exc := get_collapsed_eg(beg))
+            and
+            type(exc) not in ignore
+        ):
+
+            # TODO? report number of nested groups it was collapsed
+            # *from*?
+            if add_notes:
+                from_group_note: str = (
+                    '( ^^^ this exc was collapsed from a group ^^^ )\n'
+                )
+                if (
+                    from_group_note
+                    not in
+                    getattr(exc, "__notes__", ())
+                ):
+                    exc.add_note(from_group_note)
+
+            # raise exc
+            # ^^ this will leave the orig beg tb above with the
+            # "during the handling of <beg> the following.."
+            # So, instead do..
+            #
+            if cause := exc.__cause__:
+                raise exc from cause
+            else:
+                # suppress "during handling of <the beg>"
+                # output in tb/console.
+                raise exc from None
+
+        # keep original
+        raise # beg
+
+
+def is_multi_cancelled(
+    beg: BaseException|BaseExceptionGroup,
+
+    ignore_nested: set[BaseException] = set(),
+
+) -> Literal[False]|BaseExceptionGroup:
+    '''
+    Predicate to determine if an `BaseExceptionGroup` only contains
+    some (maybe nested) set of sub-grouped exceptions (like only
+    `trio.Cancelled`s which get swallowed silently by default) and is
+    thus the result of "gracefully cancelling" a collection of
+    sub-tasks (or other conc primitives) and receiving a "cancelled
+    ACK" from each after termination.
+
+    Docs:
+    ----
+    - https://docs.python.org/3/library/exceptions.html#exception-groups
+    - https://docs.python.org/3/library/exceptions.html#BaseExceptionGroup.subgroup
+
+    '''
+
+    if (
+        not ignore_nested
+        or
+        trio.Cancelled not in ignore_nested
+        # XXX always count-in `trio`'s native signal
+    ):
+        ignore_nested.update({trio.Cancelled})
+
+    if isinstance(beg, BaseExceptionGroup):
+        # https://docs.python.org/3/library/exceptions.html#BaseExceptionGroup.subgroup
+        # |_ "The condition can be an exception type or tuple of
+        #   exception types, in which case each exception is checked
+        #   for a match using the same check that is used in an
+        #   except clause. The condition can also be a callable
+        #   (other than a type object) that accepts an exception as
+        #   its single argument and returns true for the exceptions
+        #   that should be in the subgroup."
+        matched_exc: BaseExceptionGroup|None = beg.subgroup(
+            tuple(ignore_nested),
+
+            # ??TODO, complain about why not allowed to use
+            # named arg style calling???
+            # XD .. wtf?
+            # condition=tuple(ignore_nested),
+        )
+        if matched_exc is not None:
+            return matched_exc
+
+    # NOTE, IFF no excs types match (throughout the error-tree)
+    # -> return `False`, OW return the matched sub-eg.
+    #
+    # IOW, for the inverse of ^ for the purpose of
+    # maybe-enter-REPL--logic: "only debug when the err-tree contains
+    # at least one exc-type NOT in `ignore_nested`" ; i.e. the case where
+    # we fallthrough and return `False` here.
+    return False

tractor/trionics/_mngrs.py

@@ -31,7 +31,6 @@ from typing import (
     AsyncIterator,
     Callable,
     Hashable,
-    Optional,
     Sequence,
     TypeVar,
     TYPE_CHECKING,
@@ -40,6 +39,11 @@ from typing import (
 import trio
 from tractor._state import current_actor
 from tractor.log import get_logger
+# from ._beg import collapse_eg
+# from ._taskc import (
+#     maybe_raise_from_masking_exc,
+# )
+
 
 if TYPE_CHECKING:
     from tractor import ActorNursery
@@ -104,6 +108,9 @@ async def _enter_and_wait(
 async def gather_contexts(
     mngrs: Sequence[AsyncContextManager[T]],
 
+    # caller can provide their own scope
+    tn: trio.Nursery|None = None,
+
 ) -> AsyncGenerator[
     tuple[
         T | None,
@@ -112,17 +119,19 @@ async def gather_contexts(
     None,
 ]:
     '''
-    Concurrently enter a sequence of async context managers (acms),
-    each from a separate `trio` task and deliver the unwrapped
-    `yield`-ed values in the same order once all managers have entered.
+    Concurrently enter a sequence of async context managers (`acm`s),
+    each scheduled in a separate `trio.Task` and deliver their
+    unwrapped `yield`-ed values in the same order once all `@acm`s
+    in every task have entered.
 
-    On exit, all acms are subsequently and concurrently exited.
+    On exit, all `acm`s are subsequently and concurrently exited with
+    **no order guarantees**.
 
     This function is somewhat similar to a batch of non-blocking
     calls to `contextlib.AsyncExitStack.enter_async_context()`
     (inside a loop) *in combo with* a `asyncio.gather()` to get the
     `.__aenter__()`-ed values, except the managers are both
-    concurrently entered and exited and *cancellation just works*(R).
+    concurrently entered and exited and *cancellation-just-works™*.
 
     '''
     seed: int = id(mngrs)
@@ -142,37 +151,47 @@ async def gather_contexts(
     if not mngrs:
         raise ValueError(
             '`.trionics.gather_contexts()` input mngrs is empty?\n'
+            '\n'
             'Did try to use inline generator syntax?\n'
-            'Use a non-lazy iterator or sequence type intead!'
+            'Check that list({mngrs}) works!\n'
+            # 'or sequence-type intead!\n'
+            # 'Use a non-lazy iterator or sequence-type intead!\n'
         )
 
-    async with trio.open_nursery(
-        strict_exception_groups=False,
-        # ^XXX^ TODO? soo roll our own then ??
-        # -> since we kinda want the "if only one `.exception` then
-        # just raise that" interface?
-    ) as tn:
-        for mngr in mngrs:
-            tn.start_soon(
-                _enter_and_wait,
-                mngr,
-                unwrapped,
-                all_entered,
-                parent_exit,
-                seed,
-            )
+    try:
+        async with (
+            #
+            # ?TODO, does including these (eg-collapsing,
+            # taskc-unmasking) improve tb noise-reduction/legibility?
+            #
+            # collapse_eg(),
+            maybe_open_nursery(
+                nursery=tn,
+            ) as tn,
+            # maybe_raise_from_masking_exc(),
+        ):
+            for mngr in mngrs:
+                tn.start_soon(
+                    _enter_and_wait,
+                    mngr,
+                    unwrapped,
+                    all_entered,
+                    parent_exit,
+                    seed,
+                )
 
-        # deliver control once all managers have started up
-        await all_entered.wait()
-
-        try:
+            # deliver control to caller once all ctx-managers have
+            # started (yielded back to us).
+            await all_entered.wait()
             yield tuple(unwrapped.values())
-        finally:
-            # NOTE: this is ABSOLUTELY REQUIRED to avoid
-            # the following wacky bug:
-            # <tractorbugurlhere>
             parent_exit.set()
 
+    finally:
+        # XXX NOTE: this is ABSOLUTELY REQUIRED to avoid
+        # the following wacky bug:
+        # <tractorbugurlhere>
+        parent_exit.set()
+
 
 # Per actor task caching helpers.
 # Further potential examples of interest:
@@ -184,7 +203,7 @@ class _Cache:
     a kept-alive-while-in-use async resource.
 
     '''
-    service_n: Optional[trio.Nursery] = None
+    service_tn: trio.Nursery|None = None
     locks: dict[Hashable, trio.Lock] = {}
     users: int = 0
     values: dict[Any,  Any] = {}
@@ -193,7 +212,7 @@ class _Cache:
         tuple[trio.Nursery, trio.Event]
     ] = {}
     # nurseries: dict[int, trio.Nursery] = {}
-    no_more_users: Optional[trio.Event] = None
+    no_more_users: trio.Event|None = None
 
     @classmethod
     async def run_ctx(
@@ -203,16 +222,18 @@ class _Cache:
         task_status: trio.TaskStatus[T] = trio.TASK_STATUS_IGNORED,
 
     ) -> None:
-        async with mng as value:
-            _, no_more_users = cls.resources[ctx_key]
-            cls.values[ctx_key] = value
-            task_status.started(value)
-            try:
-                await no_more_users.wait()
-            finally:
-                # discard nursery ref so it won't be re-used (an error)?
-                value = cls.values.pop(ctx_key)
-                cls.resources.pop(ctx_key)
+        try:
+            async with mng as value:
+                _, no_more_users = cls.resources[ctx_key]
+                try:
+                    cls.values[ctx_key] = value
+                    task_status.started(value)
+                    await no_more_users.wait()
+                finally:
+                    value = cls.values.pop(ctx_key)
+        finally:
+            # discard nursery ref so it won't be re-used (an error)?
+            cls.resources.pop(ctx_key)
 
 
 @acm
@@ -225,6 +246,9 @@ async def maybe_open_context(
     kwargs: dict = {},
     key: Hashable | Callable[..., Hashable] = None,
 
+    # caller can provide their own scope
+    tn: trio.Nursery|None = None,
+
 ) -> AsyncIterator[tuple[bool, T]]:
     '''
     Maybe open an async-context-manager (acm) if there is not already
@@ -257,40 +281,94 @@ async def maybe_open_context(
     # have it not be closed until all consumers have exited (which is
     # currently difficult to implement any other way besides using our
     # pre-allocated runtime instance..)
-    service_n: trio.Nursery = current_actor()._service_n
+    if tn:
+        # TODO, assert tn is eventual parent of this task!
+        task: trio.Task = trio.lowlevel.current_task()
+        task_tn: trio.Nursery = task.parent_nursery
+        if not tn._cancel_status.encloses(
+            task_tn._cancel_status
+        ):
+            raise RuntimeError(
+                f'Mis-nesting of task under provided {tn} !?\n'
+                f'Current task is NOT a child(-ish)!!\n'
+                f'\n'
+                f'task: {task}\n'
+                f'task_tn: {task_tn}\n'
+            )
+        service_tn = tn
+    else:
+        service_tn: trio.Nursery = current_actor()._service_tn
 
     # TODO: is there any way to allocate
     # a 'stays-open-till-last-task-finshed nursery?
-    # service_n: trio.Nursery
-    # async with maybe_open_nursery(_Cache.service_n) as service_n:
-    #     _Cache.service_n = service_n
+    # service_tn: trio.Nursery
+    # async with maybe_open_nursery(_Cache.service_tn) as service_tn:
+    #     _Cache.service_tn = service_tn
 
+    cache_miss_ke: KeyError|None = None
+    maybe_taskc: trio.Cancelled|None = None
     try:
         # **critical section** that should prevent other tasks from
         # checking the _Cache until complete otherwise the scheduler
         # may switch and by accident we create more then one resource.
         yielded = _Cache.values[ctx_key]
 
-    except KeyError:
-        log.debug(f'Allocating new {acm_func} for {ctx_key}')
-        mngr = acm_func(**kwargs)
-        resources = _Cache.resources
-        assert not resources.get(ctx_key), f'Resource exists? {ctx_key}'
-        resources[ctx_key] = (service_n, trio.Event())
+    except KeyError as _ke:
+        # XXX, stay mutexed up to cache-miss yield
+        try:
+            cache_miss_ke = _ke
+            log.debug(
+                f'Allocating new @acm-func entry\n'
+                f'ctx_key={ctx_key}\n'
+                f'acm_func={acm_func}\n'
+            )
+            mngr = acm_func(**kwargs)
+            resources = _Cache.resources
+            assert not resources.get(ctx_key), f'Resource exists? {ctx_key}'
+            resources[ctx_key] = (service_tn, trio.Event())
+            yielded: Any = await service_tn.start(
+                _Cache.run_ctx,
+                mngr,
+                ctx_key,
+            )
+            _Cache.users += 1
+        finally:
+            # XXX, since this runs from an `except` it's a checkpoint
+            # whih can be `trio.Cancelled`-masked.
+            #
+            # NOTE, in that case the mutex is never released by the
+            # (first and) caching task and **we can't** simply shield
+            # bc that will inf-block on the `await
+            # no_more_users.wait()`.
+            #
+            # SO just always unlock!
+            lock.release()
 
-        # sync up to the mngr's yielded value
-        yielded = await service_n.start(
-            _Cache.run_ctx,
-            mngr,
-            ctx_key,
-        )
-        _Cache.users += 1
-        lock.release()
-        yield False, yielded
+        try:
+            yield (
+                False,  # cache_hit = "no"
+                yielded,
+            )
+        except trio.Cancelled as taskc:
+            maybe_taskc = taskc
+            log.cancel(
+                f'Cancelled from cache-miss entry\n'
+                f'\n'
+                f'ctx_key: {ctx_key!r}\n'
+                f'mngr: {mngr!r}\n'
+            )
+            # XXX, always unset ke from cancelled context
+            # since we never consider it a masked exc case!
+            # - bc this can be called directly ty `._rpc._invoke()`?
+            #
+            if maybe_taskc.__context__ is cache_miss_ke:
+                maybe_taskc.__context__ = None
+
+            raise taskc
 
     else:
         _Cache.users += 1
-        log.runtime(
+        log.debug(
             f'Re-using cached resource for user {_Cache.users}\n\n'
             f'{ctx_key!r} -> {type(yielded)}\n'
 
@@ -300,9 +378,19 @@ async def maybe_open_context(
             # f'{ctx_key!r} -> {yielded!r}\n'
         )
         lock.release()
-        yield True, yielded
+        yield (
+            True,  # cache_hit = "yes"
+            yielded,
+        )
 
     finally:
+        if lock.locked():
+            stats: trio.LockStatistics = lock.statistics()
+            log.error(
+                f'Lock left locked by last owner !?\n'
+                f'{stats}\n'
+            )
+
         _Cache.users -= 1
 
         if yielded is not None:

tractor/trionics/_taskc.py

@@ -0,0 +1,293 @@
+# tractor: structured concurrent "actors".
+# Copyright 2018-eternity Tyler Goodlet.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+`trio.Task` cancellation helpers, extensions and "holsters".
+
+'''
+from __future__ import annotations
+from contextlib import (
+    asynccontextmanager as acm,
+)
+import inspect
+from types import (
+    TracebackType,
+)
+from typing import (
+    Type,
+    TYPE_CHECKING,
+)
+
+import trio
+from tractor.log import get_logger
+
+log = get_logger(__name__)
+
+
+if TYPE_CHECKING:
+    from tractor.devx.debug import BoxedMaybeException
+
+
+def find_masked_excs(
+    maybe_masker: BaseException,
+    unmask_from: set[BaseException],
+) -> BaseException|None:
+    ''''
+    Deliver any `maybe_masker.__context__` provided
+    it a declared masking exc-type entry in `unmask_from`.
+
+    '''
+    if (
+        type(maybe_masker) in unmask_from
+        and
+        (exc_ctx := maybe_masker.__context__)
+
+        # TODO? what about any cases where
+        # they could be the same type but not same instance?
+        # |_i.e. a cancel masking a cancel ??
+        # or (
+        #     exc_ctx is not maybe_masker
+        # )
+    ):
+        return exc_ctx
+
+    return None
+
+
+_mask_cases: dict[
+    Type[Exception],  # masked exc type
+    dict[
+        int,  # inner-frame index into `inspect.getinnerframes()`
+        # `FrameInfo.function/filename: str`s to match
+        dict[str, str],
+    ],
+] = {
+    trio.WouldBlock: {
+        # `trio.Lock.acquire()` has a checkpoint inside the
+        # `WouldBlock`-no_wait path's handler..
+        -5: {  # "5th frame up" from checkpoint
+            'filename': 'trio/_sync.py',
+            'function': 'acquire',
+            # 'lineno': 605,  # matters?
+        },
+    }
+}
+
+
+def is_expected_masking_case(
+    cases: dict,
+    exc_ctx: Exception,
+    exc_match: BaseException,
+
+) -> bool|inspect.FrameInfo:
+    '''
+    Determine whether the provided masked exception is from a known
+    bug/special/unintentional-`trio`-impl case which we do not wish
+    to unmask.
+
+    Return any guilty `inspect.FrameInfo` ow `False`.
+
+    '''
+    exc_tb: TracebackType = exc_match.__traceback__
+    if cases := _mask_cases.get(type(exc_ctx)):
+        inner: list[inspect.FrameInfo] = inspect.getinnerframes(exc_tb)
+
+        # from tractor.devx.debug import mk_pdb
+        # mk_pdb().set_trace()
+        for iframe, matchon in cases.items():
+            try:
+                masker_frame: inspect.FrameInfo = inner[iframe]
+            except IndexError:
+                continue
+
+            for field, in_field in matchon.items():
+                val = getattr(
+                    masker_frame,
+                    field,
+                )
+                if in_field not in val:
+                    break
+            else:
+                return masker_frame
+
+    return False
+
+
+
+# XXX, relevant discussion @ `trio`-core,
+# https://github.com/python-trio/trio/issues/455
+#
+@acm
+async def maybe_raise_from_masking_exc(
+    unmask_from: (
+        BaseException|
+        tuple[BaseException]
+    ) = (trio.Cancelled,),
+
+    raise_unmasked: bool = True,
+    extra_note: str = (
+        'This can occurr when,\n'
+        '\n'
+        ' - a `trio.Nursery/CancelScope` embeds a `finally/except:`-block '
+        'which execs an un-shielded checkpoint!'
+        #
+        # ^TODO? other cases?
+    ),
+
+    always_warn_on: tuple[Type[BaseException]] = (
+        trio.Cancelled,
+    ),
+
+    # don't ever unmask or warn on any masking pair,
+    # {<masked-excT-key> -> <masking-excT-value>}
+    never_warn_on: dict[
+        Type[BaseException],
+        Type[BaseException],
+    ] = {
+        KeyboardInterrupt: trio.Cancelled,
+        trio.Cancelled: trio.Cancelled,
+    },
+    # ^XXX, special case(s) where we warn-log bc likely
+    # there will be no operational diff since the exc
+    # is always expected to be consumed.
+
+) -> BoxedMaybeException:
+    '''
+    Maybe un-mask and re-raise exception(s) suppressed by a known
+    error-used-as-signal type (cough namely `trio.Cancelled`).
+
+    Though this unmasker targets cancelleds, it can be used more
+    generally to capture and unwrap masked excs detected as
+    `.__context__` values which were suppressed by any error type
+    passed in `unmask_from`.
+
+    -------------
+    STILL-TODO ??
+    -------------
+    -[ ] support for egs which have multiple masked entries in
+        `maybe_eg.exceptions`, in which case we should unmask the
+        individual sub-excs but maintain the eg-parent's form right?
+
+    '''
+    if not isinstance(unmask_from, tuple):
+        raise ValueError(
+            f'Invalid unmask_from = {unmask_from!r}\n'
+            f'Must be a `tuple[Type[BaseException]]`.\n'
+        )
+
+    from tractor.devx.debug import (
+        BoxedMaybeException,
+    )
+    boxed_maybe_exc = BoxedMaybeException(
+        raise_on_exit=raise_unmasked,
+    )
+    matching: list[BaseException]|None = None
+    try:
+        yield boxed_maybe_exc
+        return
+    except BaseException as _bexc:
+        bexc = _bexc
+        if isinstance(bexc, BaseExceptionGroup):
+            matches: ExceptionGroup
+            matches, _ = bexc.split(unmask_from)
+            if matches:
+                matching = matches.exceptions
+
+        elif (
+            unmask_from
+            and
+            type(bexc) in unmask_from
+        ):
+            matching = [bexc]
+
+    if matching is None:
+        raise
+
+    masked: list[tuple[BaseException, BaseException]] = []
+    for exc_match in matching:
+        if exc_ctx := find_masked_excs(
+            maybe_masker=exc_match,
+            unmask_from=set(unmask_from),
+        ):
+            masked.append((
+                exc_ctx,
+                exc_match,
+            ))
+            boxed_maybe_exc.value = exc_match
+            note: str = (
+                f'\n'
+                f'^^WARNING^^\n'
+                f'the above {type(exc_ctx)!r} was masked by a {type(exc_match)!r}\n'
+            )
+            if extra_note:
+                note += (
+                    f'\n'
+                    f'{extra_note}\n'
+                )
+
+            do_warn: bool = (
+                never_warn_on.get(
+                    type(exc_ctx)  # masking type
+                )
+                is not
+                type(exc_match)  # masked type
+            )
+
+            if do_warn:
+                exc_ctx.add_note(note)
+
+            if (
+                do_warn
+                and
+                type(exc_match) in always_warn_on
+            ):
+                log.warning(note)
+
+            if (
+                do_warn
+                and
+                raise_unmasked
+            ):
+                if len(masked) < 2:
+                    # don't unmask already known "special" cases..
+                    if (
+                        _mask_cases
+                        and
+                        (cases := _mask_cases.get(type(exc_ctx)))
+                        and
+                        (masker_frame := is_expected_masking_case(
+                            cases,
+                            exc_ctx,
+                            exc_match,
+                        ))
+                    ):
+                        log.warning(
+                            f'Ignoring already-known, non-ideal-but-valid '
+                            f'masker code @\n'
+                            f'{masker_frame}\n'
+                            f'\n'
+                            f'NOT raising {exc_ctx} from masker {exc_match!r}\n'
+                        )
+                        raise exc_match
+
+                    raise exc_ctx from exc_match
+
+                # ??TODO, see above but, possibly unmasking sub-exc
+                # entries if there are > 1
+                # else:
+                #     await pause(shield=True)
+    else:
+        raise