Compare commits

..

6 Commits

Author SHA1 Message Date
Gud Boi feb25af8b8 .ui._editors: log multiline styling and re-leveling 2026-02-01 20:38:02 -05:00
Gud Boi 2bf3aaddac .ui._lines: drop unused graphics-item import 2026-02-01 20:30:31 -05:00
Gud Boi d78b8c4df3 Add batch-submit API for gap annotations
Introduce `AnnotCtl.add_batch()` and `serve_rc_annots()` batch
handler to submit 1000s of gaps in single IPC msg instead of
per-annot round-trips. Server builds `GapAnnotations` from specs
and handles vectorized timestamp-to-index lookups.

Deats,
- add `'cmd': 'batch'` handler in `serve_rc_annots()`
- vectorized timestamp lookup via `np.searchsorted()` + masking
- build `gap_specs: list[dict]` from rect+arrow specs client-side
- create single `GapAnnotations` item for all gaps server-side
- handle `GapAnnotations.reposition()` in redraw handler
- add profiling to batch path for perf measurement
- support optional individual arrows for A/B comparison

Also,
- refactor `markup_gaps()` to collect specs + single batch call
- add `no_qt_updates()` context mgr for batch render ops
- add profiling to annotation teardown path
- add `GapAnnotations` case to `rm_annot()` match block

(this patch was generated in some part by [`claude-code`][claude-code-gh])
[claude-code-gh]: https://github.com/anthropics/claude-code
2026-02-01 20:23:52 -05:00
Gud Boi 16c770a808 Add a `GapAnnotations` path-renderer
For a ~1000x perf gain says ol' claudy, our boi who wrote this entire
patch! Bo

Introduce `GapAnnotations` in `.ui._annotate` for batch-rendering gap
rects/arrows instead of individual `QGraphicsItem` instances. Uses
upstream's `pyqtgraph.Qt.internals.PrimitiveArray` for rects and
a `QPainterPath` for arrows. This API-replicates our prior annotator's
in view shape-graphics but now using (what we're dubbing)
"single-array-multiple-graphics" tech much like our `.ui._curve`
extensions to `pg` B)

Impl deats,
- batch draw ~1000 gaps in single paint call vs 1000 items
- arrows render in scene coords to maintain pixel size on zoom
- add vectorized timestamp-to-index lookup for repositioning
- cache bounding rect, rebuild on `reposition()` calls
- match `SelectRect` + `ArrowItem` visual style/colors
- skip reposition when timeframe doesn't match gap's period

Other,
- fix typo in `LevelMarker` docstring: "graphich" -> "graphic"
- reflow docstring in `qgo_draw_markers()` to 67 char limit

(this patch was generated in some part by [`claude-code`][claude-code-gh])
[claude-code-gh]: https://github.com/anthropics/claude-code
2026-02-01 20:10:01 -05:00
Gud Boi 191f4b5e4c Pin us to upstream `pyqtgraph` github:master 2026-02-01 20:00:40 -05:00
Gud Boi 28d0babc6d Add info log for shm processing in `ldshm` CLI cmd
Log shm file name and detected period before null segment
processing to aid debugging.

(this patch was generated in some part by [`claude-code`][claude-code-gh])
[claude-code-gh]: https://github.com/anthropics/claude-code
2026-02-01 19:44:28 -05:00
3 changed files with 13 additions and 57 deletions

View File

@ -275,15 +275,9 @@ async def open_history_client(
f'{times}'
)
# XXX, debug any case where the latest 1m bar we get is
# already another "sample's-step-old"..
if end_dt is None:
inow: int = round(time.time())
if (
_time_step := (inow - times[-1])
>
timeframe * 2
):
if (inow - times[-1]) > 60:
await tractor.pause()
start_dt = from_timestamp(times[0])

View File

@ -452,13 +452,7 @@ def ldshm(
)
# last chance manual overwrites in REPL
# await tractor.pause()
if not aids:
log.warning(
f'No gaps were found !?\n'
f'fqme: {fqme!r}\n'
f'timeframe: {period_s!r}\n'
f"WELL THAT'S GOOD NOOZ!\n"
)
assert aids
tf2aids[period_s] = aids
else:

View File

@ -49,7 +49,6 @@ from pendulum import (
Duration,
duration as mk_duration,
from_timestamp,
timezone,
)
import numpy as np
import polars as pl
@ -58,7 +57,9 @@ from piker.brokers import NoData
from piker.accounting import (
MktPair,
)
from piker.log import get_logger
from piker.data._util import (
log,
)
from ..data._sharedmem import (
maybe_open_shm_array,
ShmArray,
@ -96,9 +97,6 @@ if TYPE_CHECKING:
# from .feed import _FeedsBus
log = get_logger()
# `ShmArray` buffer sizing configuration:
_mins_in_day = int(60 * 24)
# how much is probably dependent on lifestyle
@ -403,9 +401,7 @@ async def start_backfill(
# based on the sample step size, maybe load a certain amount history
update_start_on_prepend: bool = False
if (
_until_was_none := (backfill_until_dt is None)
):
if backfill_until_dt is None:
# TODO: per-provider default history-durations?
# -[ ] inside the `open_history_client()` config allow
@ -439,8 +435,6 @@ async def start_backfill(
last_start_dt: datetime = backfill_from_dt
next_prepend_index: int = backfill_from_shm_index
est = timezone('EST')
while last_start_dt > backfill_until_dt:
log.info(
f'Requesting {timeframe}s frame:\n'
@ -454,10 +448,9 @@ async def start_backfill(
next_end_dt,
) = await get_hist(
timeframe,
end_dt=(end_dt_param := last_start_dt),
end_dt=last_start_dt,
)
except NoData as nodata:
_nodata = nodata
except NoData as _daterr:
orig_last_start_dt: datetime = last_start_dt
gap_report: str = (
f'EMPTY FRAME for `end_dt: {last_start_dt}`?\n'
@ -525,32 +518,8 @@ async def start_backfill(
==
next_start_dt.timestamp()
)
assert (
(last_time := time[-1])
==
next_end_dt.timestamp()
)
frame_last_dt = from_timestamp(last_time)
if (
frame_last_dt.add(seconds=timeframe)
<
end_dt_param
):
est_frame_last_dt = est.convert(frame_last_dt)
est_end_dt_param = est.convert(end_dt_param)
log.warning(
f'Provider frame ending BEFORE requested end_dt={end_dt_param} ??\n'
f'frame_last_dt (EST): {est_frame_last_dt!r}\n'
f'end_dt_param (EST): {est_end_dt_param!r}\n'
f'\n'
f'Likely contains,\n'
f'- a venue closure.\n'
f'- (maybe?) missing data ?\n'
)
# ?TODO, check against venue closure hours
# if/when provided by backend?
await tractor.pause()
assert time[-1] == next_end_dt.timestamp()
expected_dur: Interval = (
last_start_dt.subtract(
@ -612,11 +581,10 @@ async def start_backfill(
'0 BARS TO PUSH after diff!?\n'
f'{next_start_dt} -> {last_start_dt}'
)
await tractor.pause()
# Check if we're about to exceed buffer capacity BEFORE
# attempting the push
if (next_prepend_index - ln) < 0:
if next_prepend_index - ln < 0:
log.warning(
f'Backfill would exceed buffer capacity!\n'
f'next_prepend_index: {next_prepend_index}\n'
@ -687,7 +655,7 @@ async def start_backfill(
},
})
# XXX, can't push the entire frame? so
# can't push the entire frame? so
# push only the amount that can fit..
break
@ -747,8 +715,8 @@ async def start_backfill(
) = dedupe(df)
if diff:
log.warning(
f'Found {diff!r} duplicates in tsdb! '
f'=> Overwriting with `deduped` data !! <=\n'
f'Found {diff} duplicates in tsdb, '
f'overwriting with deduped data\n'
)
await storage.write_ohlcv(
col_sym_key,