diff --git a/.claude/skills/run-tests/test-harness-reference.md b/.claude/skills/run-tests/test-harness-reference.md index 8ef3e0cf..4bb0c601 100644 --- a/.claude/skills/run-tests/test-harness-reference.md +++ b/.claude/skills/run-tests/test-harness-reference.md @@ -166,8 +166,13 @@ whole suite supports UDS merely because the Tractor plugin exposes it. before fixtures can isolate config. If explicitly requested, isolate `XDG_CONFIG_HOME` before Python starts and use the proper Qt/Nix shell. - Piker and the installed Tractor pytest plugin do not provide a - repository-local process, shared-memory, or socket reaper. Never apply - historical broad `pkill -f tractor._child` guidance automatically. + repository-local process or socket reaper. Never apply historical broad + `pkill -f tractor._child` guidance automatically. +- The function-scoped autouse `shm_leak_tracker` fixture wraps Tractor's + current-process `SharedMemory` factory. It tracks only successful + `create=True` calls, restores the pre-test `_known_tokens` cache, and + unlinks exact surviving names before failing the leaking test. It never + scans `/dev/shm` or unlinks attachments created by another process. ## Test Layout diff --git a/ai/prompt-io/opencode/20260727T215906Z_689df816_prompt_io.md b/ai/prompt-io/opencode/20260727T215906Z_689df816_prompt_io.md new file mode 100644 index 00000000..b8ae1853 --- /dev/null +++ b/ai/prompt-io/opencode/20260727T215906Z_689df816_prompt_io.md @@ -0,0 +1,41 @@ +--- +model: gpt-5.6-sol +provider: openai +service: opencode +session: ses_0799212ebffe42arY96czXn89F +timestamp: 2026-07-27T21:59:06Z +git_ref: 689df816 +scope: code +substantive: true +raw_file: 20260727T215906Z_689df816_prompt_io.raw.md +--- + +## Prompt + +After the actor/SHM/Parquet integration test exposed a stale SHM triplet, +the user requested leak-cleaner machinery in the pytest harness using +Tractor's existing ownership and lifetime patterns. + +## Response summary + +Added creator-scoped SHM leak tracking to pytest without scanning +`/dev/shm` or treating attachments as owned. The fixture cooperates with +normal Tractor teardown, verifies POSIX object identity before fallback +cleanup, restores process-local token state, and fails after cleaning a +leak. Regressions cover the exact pre-actor-registration allocation window, +positional creators, and external attachment safety. + +## Files changed + +- `tests/conftest.py` - creator-scoped SHM leak tracking fixture +- `tests/test_shm_cleanup.py` - ownership and failure-window regressions +- `.claude/skills/run-tests/test-harness-reference.md` + - document current-process SHM cleanup behavior +- `ai/prompt-io/opencode/20260727T215906Z_689df816_prompt_io.raw.md` + - unedited response record +- `ai/prompt-io/opencode/20260727T215906Z_689df816_prompt_io.md` + - provenance metadata and response summary + +## Human edits + +None - generated changes have not been edited by the human. diff --git a/ai/prompt-io/opencode/20260727T215906Z_689df816_prompt_io.raw.md b/ai/prompt-io/opencode/20260727T215906Z_689df816_prompt_io.raw.md new file mode 100644 index 00000000..869ff1e1 --- /dev/null +++ b/ai/prompt-io/opencode/20260727T215906Z_689df816_prompt_io.raw.md @@ -0,0 +1,25 @@ +--- +model: gpt-5.6-sol +provider: openai +service: opencode +timestamp: 2026-07-27T21:59:06Z +git_ref: 689df816 +diff_cmd: git diff HEAD~1..HEAD +--- + +> `git diff HEAD~1..HEAD -- tests/conftest.py tests/test_shm_cleanup.py .claude/skills/run-tests/test-harness-reference.md` + +Added function-scoped SHM ownership tracking around Tractor's test-process +factory. The fixture records only successful creators, removes ownership +records during normal Tractor teardown, verifies POSIX object identity +before fallback unlink, restores the token-cache baseline, and fails after +cleaning any leak. Regressions reproduce allocation failure before actor +lifetime registration and prove external attachments remain untouched. + +Verification generated with the patch: + +- SHM cleanup regressions: 2 passed +- combined SHM, NativeDB, and history set: 25 passed +- Ruff: passed +- `git diff --check`: passed +- final adversarial review: no findings diff --git a/tests/conftest.py b/tests/conftest.py index 599cf568..ad7ecce4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,8 +1,13 @@ from contextlib import asynccontextmanager as acm +from collections.abc import Callable from functools import partial import logging import os from pathlib import Path +from weakref import ( + ReferenceType, + ref, +) import pytest import tractor @@ -89,6 +94,151 @@ def log( ) +@pytest.fixture(autouse=True) +def shm_leak_tracker( + monkeypatch: pytest.MonkeyPatch, + +) -> Callable[[], set[str]]: + ''' + Clean SHM created but not released by the current test process. + + Tractor's token cache includes attachments, so cache membership + can not prove ownership. Track only successful ``create=True`` + calls through Tractor's own factory. Normal actor-stack teardown + unlinks these names first; any survivors are exact test-owned leaks + which this fixture removes before failing the test. + + ''' + from tractor.ipc import _shm + + known_tokens: dict = dict(_shm._known_tokens) + owned_segments: dict[ + str, + tuple[ + tuple[int, int]|None, + ReferenceType, + ], + ] = {} + open_shm = _shm.SharedMemory + unlink_shm = getattr(_shm, 'shm_unlink', None) + + def normalize_name(name: str) -> str: + return str(name).lstrip('/') + + def segment_id(segment) -> tuple[int, int]|None: + fd: int = getattr(segment, '_fd', -1) + if fd < 0: + return None + stat = os.fstat(fd) + return stat.st_dev, stat.st_ino + + def open_test_shm(*args, **kwargs): + segment = open_shm(*args, **kwargs) + create: bool = kwargs.get( + 'create', + args[1] if len(args) > 1 else False, + ) + if create: + name: str = normalize_name(segment.name) + owned_segments[name] = ( + segment_id(segment), + ref(segment), + ) + return segment + + if unlink_shm is not None: + def unlink_owned_name(name: str) -> None: + normalized: str = normalize_name(name) + try: + unlink_shm(name) + except FileNotFoundError: + owned_segments.pop(normalized, None) + raise + else: + owned_segments.pop(normalized, None) + + monkeypatch.setattr( + _shm, + 'shm_unlink', + unlink_owned_name, + ) + + def cleanup() -> set[str]: + leaked: set[str] = set() + errors: list[Exception] = [] + try: + for name, record in list(owned_segments.items()): + expected_id, segment_ref = record + try: + segment = open_shm( + name=name, + create=False, + ) + except FileNotFoundError: + segment = None + except Exception as err: + errors.append(err) + segment = None + else: + try: + actual_id: tuple[int, int]|None = segment_id( + segment + ) + if expected_id is None: + errors.append(RuntimeError( + f'Can not verify SHM ownership: {name}' + )) + elif actual_id != expected_id: + errors.append(RuntimeError( + f'SHM name changed ownership: {name}' + )) + else: + leaked.add(name) + segment.unlink() + except FileNotFoundError: + pass + except Exception as err: + errors.append(err) + finally: + try: + segment.close() + except Exception as err: + errors.append(err) + + original = segment_ref() + if original is not None: + try: + original.close() + except Exception as err: + errors.append(err) + finally: + owned_segments.clear() + _shm._known_tokens.clear() + _shm._known_tokens.update(known_tokens) + + if errors: + raise ExceptionGroup( + 'Failed to clean test-owned SHM', + errors, + ) + return leaked + + monkeypatch.setattr( + _shm, + 'SharedMemory', + open_test_shm, + ) + yield cleanup + + leaked: set[str] = cleanup() + if leaked: + names: str = '\n'.join(sorted(leaked)) + pytest.fail( + f'Test leaked SHM segments; cleaned:\n' + f'{names}' + ) + + @acm async def _open_test_pikerd( tmpconfdir: str, diff --git a/tests/test_shm_cleanup.py b/tests/test_shm_cleanup.py new file mode 100644 index 00000000..e3b9b456 --- /dev/null +++ b/tests/test_shm_cleanup.py @@ -0,0 +1,109 @@ +''' +Pytest shared-memory ownership and leak-cleanup regressions. + +''' +from collections.abc import Callable +from uuid import uuid4 + +import numpy as np +import pytest +from tractor._exceptions import NoRuntime +from tractor.ipc import _shm +from tractor.ipc._mp_bs import disable_mantracker + +from piker.data._sharedmem import maybe_open_shm_array +from piker.data._source import def_iohlcv_fields + + +def test_shm_tracker_catches_pre_actor_failure( + shm_leak_tracker: Callable[[], set[str]], +) -> None: + ''' + Clean allocations which fail before actor-lifetime registration. + + ``open_shm_ndarray()`` creates its data and index segments before it + asks ``current_actor()`` for the lifetime stack. Without a runtime, + that lookup raises after all three OS names exist and the old harness + leaked them. Reproduce that ordering through Piker's real wrapper, + invoke tracked cleanup, and prove the data, first, and last segments + plus their process-local token are all removed. + + ''' + key = f'test_shm_failure_{uuid4().hex}' + with pytest.raises(NoRuntime): + maybe_open_shm_array( + key=key, + size=4, + dtype=np.dtype(def_iohlcv_fields), + ) + + token = _shm.NDToken.from_msg(_shm._known_tokens[key]) + names = { + token.shm_name, + token.shm_first_index_name, + token.shm_last_index_name, + } + assert shm_leak_tracker() == names + assert key not in _shm._known_tokens + for name in names: + with pytest.raises(FileNotFoundError): + _shm.SharedMemory( + name=name, + create=False, + ) + + +def test_shm_leak_tracker_unlinks_only_owned_segments( + shm_leak_tracker: Callable[[], set[str]], +) -> None: + ''' + Test cleanup must unlink creators without harming attachments. + + A failed SHM allocation can escape before Tractor registers actor + lifetime callbacks. Broad cache or filesystem cleanup is unsafe + because tests may attach to a live Piker segment they do not own. + Create one external segment through an unpatched factory, attach to + it through Tractor, and create one test-owned segment through the + tracked factory. Invoke cleanup and prove only the owned name was + removed while the external segment remains attachable. + + ''' + open_external_shm = disable_mantracker() + external = None + attachment = None + owned = None + try: + external = open_external_shm( + create=True, + size=8, + ) + attachment = _shm.SharedMemory( + name=external.name, + create=False, + ) + owned = _shm.SharedMemory(None, True, 8) + cleaned: set[str] = shm_leak_tracker() + + assert cleaned == {owned.name} + with pytest.raises(FileNotFoundError): + _shm.SharedMemory( + name=owned.name, + create=False, + ) + + survivor = _shm.SharedMemory( + name=external.name, + create=False, + ) + survivor.close() + finally: + if owned is not None: + owned.close() + if attachment is not None: + attachment.close() + if external is not None: + try: + external.unlink() + except FileNotFoundError: + pass + external.close()