Merge pull request 'port_to_latest_tractor'

#45 from port_to_latest_tractor into main
Reviewed-on: https://www.pikers.dev/pikers/piker/pulls/45

config/conf.toml

@@ -1,7 +1,9 @@
 [network]
-tsdb.backend = 'marketstore'
+pikerd = [
-tsdb.host = 'localhost'
+  '/ipv4/127.0.0.1/tcp/6116',  # std localhost daemon-actor tree
-tsdb.grpc_port = 5995
+  # '/uds/6116',  # TODO std uds socket file
+]
+
 
 [ui]
 # set custom font + size which will scale entire UI

examples/basic_order_bot.py

@@ -121,6 +121,7 @@ async def bot_main():
             # tick_throttle=10,
         ) as feed,
 
+        tractor.trionics.collapse_eg(),
         trio.open_nursery() as tn,
     ):
         assert accounts

piker/accounting/__init__.py

@@ -42,7 +42,6 @@ from ._mktinfo import (
     dec_digits,
     digits_to_dec,
     MktPair,
-    Symbol,
     unpack_fqme,
     _derivs as DerivTypes,
 )
@@ -60,7 +59,6 @@ __all__ = [
     'Asset',
     'MktPair',
     'Position',
-    'Symbol',
     'Transaction',
     'TransactionLedger',
     'dec_digits',

piker/accounting/_ledger.py

@@ -40,7 +40,7 @@ import tomli_w  # for fast ledger writing
 
 from piker.types import Struct
 from piker import config
-from ..log import get_logger
+from piker.log import get_logger
 from .calc import (
     iter_by_dt,
 )
@@ -239,7 +239,9 @@ class TransactionLedger(UserDict):
 
         symcache: SymbologyCache = self._symcache
         towrite: dict[str, Any] = {}
-        for tid, txdict in self.tx_sort(self.data.copy()):
+        for tid, txdict in self.tx_sort(
+            self.data.copy()
+        ):
             # write blank-str expiry for non-expiring assets
             if (
                 'expiry' in txdict
@@ -377,7 +379,7 @@ def open_trade_ledger(
         account,
         dirpath=_fp,
     )
-    cpy = ledger_dict.copy()
+    cpy: dict = ledger_dict.copy()
 
     # XXX NOTE: if not provided presume we are being called from
     # sync code and need to maybe run `trio` to generate..
@@ -406,7 +408,13 @@ def open_trade_ledger(
         account=account,
         mod=mod,
         symcache=symcache,
-        tx_sort=getattr(mod, 'tx_sort', tx_sort),
+
+        # NOTE: allow backends to provide custom ledger sorting
+        tx_sort=getattr(
+            mod,
+            'tx_sort',
+            tx_sort,
+        ),
     )
     try:
         yield ledger

piker/accounting/_mktinfo.py

@@ -305,8 +305,8 @@ class MktPair(Struct, frozen=True):
     # config right?
     # src_type: AssetTypeName
 
-    # for derivs, info describing contract, egs.
+    # for derivs, info describing contract, egs. strike price, call
-    # strike price, call or put, swap type, exercise model, etc.
+    # or put, swap type, exercise model, etc.
     contract_info: list[str] | None = None
 
     # TODO: rename to sectype since all of these can
@@ -677,90 +677,3 @@ def unpack_fqme(
         # '.'.join([mkt_ep, venue]),
         suffix,
     )
-
-
-class Symbol(Struct):
-    '''
-    I guess this is some kinda container thing for dealing with
-    all the different meta-data formats from brokers?
-
-    '''
-    key: str
-
-    broker: str = ''
-    venue: str = ''
-
-    # precision descriptors for price and vlm
-    tick_size: Decimal = Decimal('0.01')
-    lot_tick_size: Decimal = Decimal('0.0')
-
-    suffix: str = ''
-    broker_info: dict[str, dict[str, Any]] = {}
-
-    @classmethod
-    def from_fqme(
-        cls,
-        fqsn: str,
-        info: dict[str, Any],
-
-    ) -> Symbol:
-        broker, mktep, venue, suffix = unpack_fqme(fqsn)
-        tick_size = info.get('price_tick_size', 0.01)
-        lot_size = info.get('lot_tick_size', 0.0)
-
-        return Symbol(
-            broker=broker,
-            key=mktep,
-            tick_size=tick_size,
-            lot_tick_size=lot_size,
-            venue=venue,
-            suffix=suffix,
-            broker_info={broker: info},
-        )
-
-    @property
-    def type_key(self) -> str:
-        return list(self.broker_info.values())[0]['asset_type']
-
-    @property
-    def tick_size_digits(self) -> int:
-        return float_digits(self.tick_size)
-
-    @property
-    def lot_size_digits(self) -> int:
-        return float_digits(self.lot_tick_size)
-
-    @property
-    def price_tick(self) -> Decimal:
-        return Decimal(str(self.tick_size))
-
-    @property
-    def size_tick(self) -> Decimal:
-        return Decimal(str(self.lot_tick_size))
-
-    @property
-    def broker(self) -> str:
-        return list(self.broker_info.keys())[0]
-
-    @property
-    def fqme(self) -> str:
-        return maybe_cons_tokens([
-            self.key,  # final "pair name" (eg. qqq[/usd], btcusdt)
-            self.venue,
-            self.suffix,  # includes expiry and other con info
-            self.broker,
-        ])
-
-    def quantize(
-        self,
-        size: float,
-    ) -> Decimal:
-        digits = float_digits(self.lot_tick_size)
-        return Decimal(size).quantize(
-            Decimal(f'1.{"0".ljust(digits, "0")}'),
-            rounding=ROUND_HALF_EVEN
-        )
-
-    # NOTE: when cast to `str` return fqme
-    def __str__(self) -> str:
-        return self.fqme

piker/accounting/_pos.py

@@ -30,7 +30,8 @@ from types import ModuleType
 from typing import (
     Any,
     Iterator,
-    Generator
+    Generator,
+    TYPE_CHECKING,
 )
 
 import pendulum
@@ -59,8 +60,10 @@ from ..clearing._messages import (
     BrokerdPosition,
 )
 from piker.types import Struct
+from piker.log import get_logger
+
+if TYPE_CHECKING:
     from piker.data._symcache import SymbologyCache
-from ..log import get_logger
 
 log = get_logger(__name__)
 
@@ -362,7 +365,11 @@ class Position(Struct):
         # added: bool = False
         tid: str = t.tid
         if tid in self._events:
-            log.warning(f'{t} is already added?!')
+            log.debug(
+                f'Txn is already added?\n'
+                f'\n'
+                f'{t}\n'
+            )
             # return added
 
         # TODO: apparently this IS possible with a dict but not
@@ -493,6 +500,17 @@ class Account(Struct):
 
         _mktmap_table: dict[str, MktPair] | None = None,
 
+        only_require: list[str]|True = True,
+        # ^list of fqmes that are "required" to be processed from
+        # this ledger pass; we often don't care about others and
+        # definitely shouldn't always error in such cases.
+        # (eg. broker backend loaded that doesn't yet supsport the
+        # symcache but also, inside the paper engine we don't ad-hoc
+        # request `get_mkt_info()` for every symbol in the ledger,
+        # only the one for which we're simulating against).
+        # TODO, not sure if there's a better soln for this, ideally
+        # all backends get symcache support afap i guess..
+
     ) -> dict[str, Position]:
         '''
         Update the internal `.pps[str, Position]` table from input
@@ -535,11 +553,32 @@ class Account(Struct):
                 if _mktmap_table is None:
                     raise
 
+                required: bool = (
+                    only_require is True
+                    or (
+                        only_require is not True
+                        and
+                        fqme in only_require
+                    )
+                )
                 # XXX: caller is allowed to provide a fallback
                 # mktmap table for the case where a new position is
                 # being added and the preloaded symcache didn't
                 # have this entry prior (eg. with frickin IB..)
-                mkt = _mktmap_table[fqme]
+                if (
+                    not (mkt := _mktmap_table.get(fqme))
+                    and
+                    required
+                ):
+                    raise
+
+                elif not required:
+                    continue
+
+                else:
+                    # should be an entry retreived somewhere
+                    assert mkt
+
 
             if not (pos := pps.get(bs_mktid)):
 
@@ -656,7 +695,7 @@ class Account(Struct):
     def write_config(self) -> None:
         '''
         Write the current account state to the user's account TOML file, normally
-        something like ``pps.toml``.
+        something like `pps.toml`.
 
         '''
         # TODO: show diff output?
@@ -696,7 +735,7 @@ class Account(Struct):
                     else:
                         # TODO: we reallly need a diff set of
                         # loglevels/colors per subsys.
-                        log.warning(
+                        log.debug(
                             f'Recent position for {fqme} was closed!'
                         )
 

piker/accounting/calc.py

@@ -251,10 +251,16 @@ def iter_by_dt(
         for k in parsers:
             if (
                 isdict and k in tx
-                 or getattr(tx, k, None)
+                or
+                getattr(tx, k, None)
             ):
-                v = tx[k] if isdict else tx.dt
+                v = (
-                assert v is not None, f'No valid value for `{k}`!?'
+                    tx[k] if isdict
+                    else tx.dt
+                )
+                assert v is not None, (
+                    f'No valid value for `{k}`!?'
+                )
 
                 # only call parser on the value if not None from
                 # the `parsers` table above (when NOT using
@@ -269,8 +275,21 @@ def iter_by_dt(
                     return v
 
         else:
-            # XXX: should never get here..
+            # TODO: move to top?
-            breakpoint()
+            from piker.log import get_logger
+            log = get_logger(__name__)
+
+            # XXX: we should really never get here..
+            # only if a ledger record has no expected sort(able)
+            # field will we likely hit this.. like with ze IB.
+            # if no sortable field just deliver epoch?
+            log.warning(
+                'No (time) sortable field for TXN:\n'
+                f'{tx}\n'
+            )
+            return from_timestamp(0)
+            # breakpoint()
+
 
     entry: tuple[str, dict] | Transaction
     for entry in sorted(
@@ -446,7 +465,7 @@ def ledger_to_dfs(
 
         df = dfs[key] = ldf.with_columns([
 
-            pl.cumsum('size').alias('cumsize'),
+            pl.cum_sum('size').alias('cumsize'),
 
             # amount of source asset "sent" (via buy txns in
             # the market) to acquire the dst asset, PER txn.
@@ -461,7 +480,7 @@ def ledger_to_dfs(
         ]).with_columns([
 
             # rolling balance in src asset units
-            (pl.col('dst_bot').cumsum() * -1).alias('src_balance'),
+            (pl.col('dst_bot').cum_sum() * -1).alias('src_balance'),
 
             # "position operation type" in terms of increasing the
             # amount in the dst asset (entering) or decreasing the
@@ -603,7 +622,7 @@ def ledger_to_dfs(
                 # cost that was included in the least-recently
                 # entered txn that is still part of the current CSi
                 # set.
-                # => we look up the cost-per-unit cumsum and apply
+                # => we look up the cost-per-unit cum_sum and apply
                 # if over the current txn size (by multiplication)
                 # and then reverse that previusly applied cost on
                 # the txn_cost for this record.

piker/accounting/cli.py

@@ -300,7 +300,8 @@ def disect(
         assert not df.is_empty()
 
         # muck around in pdbp REPL
-        breakpoint()
+        # tractor.devx.mk_pdb().set_trace()
+        # breakpoint()
 
         # TODO: we REALLY need a better console REPL for this
         # kinda thing..

piker/brokers/_daemon.py

@@ -96,7 +96,10 @@ async def _setup_persistent_brokerd(
     # - `open_symbol_search()`
     # NOTE: see ep invocation details inside `.data.feed`.
     try:
-        async with trio.open_nursery() as service_nursery:
+        async with (
+            tractor.trionics.collapse_eg(),
+            trio.open_nursery() as service_nursery
+        ):
             bus: _FeedsBus = feed.get_feed_bus(
                 brokername,
                 service_nursery,

piker/brokers/binance/broker.py

@@ -440,6 +440,7 @@ async def open_trade_dialog(
             #   - ledger: TransactionLedger
 
             async with (
+                tractor.trionics.collapse_eg(),
                 trio.open_nursery() as tn,
                 ctx.open_stream() as ems_stream,
             ):

piker/brokers/binance/feed.py

@@ -448,7 +448,6 @@ async def subscribe(
 
 
 async def stream_quotes(
-
     send_chan: trio.abc.SendChannel,
     symbols: list[str],
     feed_is_live: trio.Event,
@@ -460,6 +459,7 @@ async def stream_quotes(
 ) -> None:
 
     async with (
+        tractor.trionics.maybe_raise_from_masking_exc(),
         send_chan as send_chan,
         open_cached_client('binance') as client,
     ):

piker/brokers/deribit/api.py

@@ -31,7 +31,7 @@ from typing import (
     Callable,
 )
 
-import pendulum
+from pendulum import now
 import trio
 from trio_typing import TaskStatus
 from rapidfuzz import process as fuzzy
@@ -39,6 +39,7 @@ import numpy as np
 from tractor.trionics import (
     broadcast_receiver,
     maybe_open_context
+    collapse_eg,
 )
 from tractor import to_asyncio
 # XXX WOOPS XD
@@ -432,6 +433,7 @@ async def get_client(
 ) -> Client:
 
     async with (
+        collapse_eg(),
         trio.open_nursery() as n,
         open_jsonrpc_session(
             _testnet_ws_url, dtype=JSONRPCResult) as json_rpc

piker/brokers/ib/api.py

@@ -48,6 +48,7 @@ from bidict import bidict
 import trio
 import tractor
 from tractor import to_asyncio
+from tractor import trionics
 from pendulum import (
     from_timestamp,
     DateTime,
@@ -1369,8 +1370,8 @@ async def load_clients_for_trio(
     '''
     Pure async mngr proxy to ``load_aio_clients()``.
 
-    This is a bootstrap entrypoing to call from
+    This is a bootstrap entrypoint to call from
-    a ``tractor.to_asyncio.open_channel_from()``.
+    a `tractor.to_asyncio.open_channel_from()`.
 
     '''
     async with load_aio_clients(
@@ -1391,7 +1392,10 @@ async def open_client_proxies() -> tuple[
     async with (
         tractor.trionics.maybe_open_context(
             acm_func=tractor.to_asyncio.open_channel_from,
-            kwargs={'target': load_clients_for_trio},
+            kwargs={
+                'target': load_clients_for_trio,
+                # ^XXX, kwarg to `open_channel_from()`
+            },
 
             # lock around current actor task access
             # TODO: maybe this should be the default in tractor?
@@ -1584,7 +1588,8 @@ async def open_client_proxy(
             event_consumers=event_table,
         ) as (first, chan),
 
-        trio.open_nursery() as relay_n,
+        trionics.collapse_eg(),  # loose-ify
+        trio.open_nursery() as relay_tn,
     ):
 
         assert isinstance(first, Client)
@@ -1624,7 +1629,7 @@ async def open_client_proxy(
 
                     continue
 
-        relay_n.start_soon(relay_events)
+        relay_tn.start_soon(relay_events)
 
         yield proxy
 

piker/brokers/ib/broker.py

@@ -34,6 +34,7 @@ import trio
 from trio_typing import TaskStatus
 import tractor
 from tractor.to_asyncio import LinkedTaskChannel
+from tractor import trionics
 from ib_insync.contract import (
     Contract,
 )
@@ -407,7 +408,7 @@ async def update_and_audit_pos_msg(
 
     # TODO: make this a "propaganda" log level?
     if ibpos.avgCost != msg.avg_price:
-        log.warning(
+        log.debug(
             f'IB "FIFO" avg price for {msg.symbol} is DIFF:\n'
             f'ib: {ibfmtmsg}\n'
             '---------------------------\n'
@@ -738,7 +739,7 @@ async def open_trade_dialog(
                                 f'UNEXPECTED POSITION says IB => {msg.symbol}\n'
                                 'Maybe they LIQUIDATED YOU or your ledger is wrong?\n'
                             )
-                            log.error(logmsg)
+                            log.debug(logmsg)
 
             await ctx.started((
                 all_positions,
@@ -747,21 +748,22 @@ async def open_trade_dialog(
 
             async with (
                 ctx.open_stream() as ems_stream,
-                trio.open_nursery() as n,
+                trionics.collapse_eg(),
+                trio.open_nursery() as tn,
             ):
                 # relay existing open orders to ems
                 for msg in order_msgs:
                     await ems_stream.send(msg)
 
                 for client in set(aioclients.values()):
-                    trade_event_stream: LinkedTaskChannel = await n.start(
+                    trade_event_stream: LinkedTaskChannel = await tn.start(
                         open_trade_event_stream,
                         client,
                     )
 
                     # start order request handler **before** local trades
                     # event loop
-                    n.start_soon(
+                    tn.start_soon(
                         handle_order_requests,
                         ems_stream,
                         accounts_def,
@@ -769,7 +771,7 @@ async def open_trade_dialog(
                     )
 
                     # allocate event relay tasks for each client connection
-                    n.start_soon(
+                    tn.start_soon(
                         deliver_trade_events,
 
                         trade_event_stream,

piker/clearing/_client.py

@@ -25,7 +25,10 @@ from typing import TYPE_CHECKING
 
 import trio
 import tractor
-from tractor.trionics import broadcast_receiver
+from tractor.trionics import (
+    broadcast_receiver,
+    collapse_eg,
+)
 
 from ._util import (
     log,  # sub-sys logger
@@ -285,8 +288,11 @@ async def open_ems(
             client._ems_stream = trades_stream
 
             # start sync code order msg delivery task
-            async with trio.open_nursery() as n:
+            async with (
-                n.start_soon(
+                collapse_eg(),
+                trio.open_nursery() as tn,
+            ):
+                tn.start_soon(
                     relay_orders_from_sync_code,
                     client,
                     fqme,
@@ -302,4 +308,4 @@ async def open_ems(
                 )
 
                 # stop the sync-msg-relay task on exit.
-                n.cancel_scope.cancel()
+                tn.cancel_scope.cancel()

piker/clearing/_ems.py

@@ -42,6 +42,7 @@ from bidict import bidict
 import trio
 from trio_typing import TaskStatus
 import tractor
+from tractor import trionics
 
 from ._util import (
     log,  # sub-sys logger
@@ -76,7 +77,6 @@ if TYPE_CHECKING:
 
 # TODO: numba all of this
 def mk_check(
-
     trigger_price: float,
     known_last: float,
     action: str,
@@ -162,7 +162,7 @@ async def clear_dark_triggers(
 
     router: Router,
     brokerd_orders_stream: tractor.MsgStream,
-    quote_stream: tractor.ReceiveMsgStream,  # noqa
+    quote_stream: tractor.MsgStream,
     broker: str,
     fqme: str,
 
@@ -178,6 +178,7 @@ async def clear_dark_triggers(
     '''
     # XXX: optimize this for speed!
     # TODO:
+    # - port to the new ringbuf stuff in `tractor.ipc`!
     # - numba all this!
     # - this stream may eventually contain multiple symbols
     quote_stream._raise_on_lag = False
@@ -500,7 +501,7 @@ class Router(Struct):
 
     '''
     # setup at actor spawn time
-    nursery: trio.Nursery
+    _tn: trio.Nursery
 
     # broker to book map
     books: dict[str, DarkBook] = {}
@@ -670,7 +671,7 @@ class Router(Struct):
                 # dark book clearing loop, also lives with parent
                 # daemon to allow dark order clearing while no
                 # client is connected.
-                self.nursery.start_soon(
+                self._tn.start_soon(
                     clear_dark_triggers,
                     self,
                     relay.brokerd_stream,
@@ -693,7 +694,7 @@ class Router(Struct):
 
                 # spawn a ``brokerd`` order control dialog stream
                 # that syncs lifetime with the parent `emsd` daemon.
-                self.nursery.start_soon(
+                self._tn.start_soon(
                     translate_and_relay_brokerd_events,
                     broker,
                     relay.brokerd_stream,
@@ -767,10 +768,12 @@ async def _setup_persistent_emsd(
 
     global _router
 
-    # open a root "service nursery" for the ``emsd`` actor
+    # open a root "service task-nursery" for the `emsd`-actor
-    async with trio.open_nursery() as service_nursery:
+    async with (
-
+        trionics.collapse_eg(),
-        _router = Router(nursery=service_nursery)
+        trio.open_nursery() as tn
+    ):
+        _router = Router(_tn=tn)
 
         # TODO: send back the full set of persistent
         # orders/execs?
@@ -1190,12 +1193,16 @@ async def process_client_order_cmds(
     submitting live orders immediately if requested by the client.
 
     '''
-    # cmd: dict
+    # TODO, only allow `msgspec.Struct` form!
+    cmd: dict
     async for cmd in client_order_stream:
-        log.info(f'Received order cmd:\n{pformat(cmd)}')
+        log.info(
+            f'Received order cmd:\n'
+            f'{pformat(cmd)}\n'
+        )
 
         # CAWT DAMN we need struct support!
-        oid = str(cmd['oid'])
+        oid: str = str(cmd['oid'])
 
         # register this stream as an active order dialog (msg flow) for
         # this order id such that translated message from the brokerd
@@ -1301,7 +1308,7 @@ async def process_client_order_cmds(
             case {
                 'oid': oid,
                 'symbol': fqme,
-                'price': trigger_price,
+                'price': price,
                 'size': size,
                 'action': ('buy' | 'sell') as action,
                 'exec_mode': ('live' | 'paper'),
@@ -1333,7 +1340,7 @@ async def process_client_order_cmds(
 
                     symbol=sym,
                     action=action,
-                    price=trigger_price,
+                    price=price,
                     size=size,
                     account=req.account,
                 )
@@ -1355,7 +1362,11 @@ async def process_client_order_cmds(
                 # (``translate_and_relay_brokerd_events()`` above) will
                 # handle relaying the ems side responses back to
                 # the client/cmd sender from this request
-                log.info(f'Sending live order to {broker}:\n{pformat(msg)}')
+                log.info(
+                    f'Sending live order to {broker}:\n'
+                    f'{pformat(msg)}'
+                )
+
                 await brokerd_order_stream.send(msg)
 
                 # an immediate response should be ``BrokerdOrderAck``
@@ -1371,7 +1382,7 @@ async def process_client_order_cmds(
             case {
                 'oid': oid,
                 'symbol': fqme,
-                'price': trigger_price,
+                'price': price,
                 'size': size,
                 'exec_mode': exec_mode,
                 'action': action,
@@ -1399,7 +1410,12 @@ async def process_client_order_cmds(
                 if isnan(last):
                     last = flume.rt_shm.array[-1]['close']
 
-                pred = mk_check(trigger_price, last, action)
+                trigger_price: float = float(price)
+                pred = mk_check(
+                    trigger_price,
+                    last,
+                    action,
+                )
 
                 # NOTE: for dark orders currently we submit
                 # the triggered live order at a price 5 ticks
@@ -1506,7 +1522,7 @@ async def maybe_open_trade_relays(
         loglevel: str = 'info',
     ):
 
-        fqme, relay, feed, client_ready = await _router.nursery.start(
+        fqme, relay, feed, client_ready = await _router._tn.start(
             _router.open_trade_relays,
             fqme,
             exec_mode,

piker/clearing/_messages.py

@@ -19,6 +19,7 @@ Clearing sub-system message and protocols.
 
 """
 from __future__ import annotations
+from decimal import Decimal
 from typing import (
     Literal,
 )
@@ -71,7 +72,15 @@ class Order(Struct):
     symbol: str  # | MktPair
     account: str  # should we set a default as '' ?
 
-    price: float
+    # https://docs.python.org/3/library/decimal.html#decimal-objects
+    #
+    # ?TODO? decimal usage throughout?
+    # -[ ] possibly leverage the `Encoder(decimal_format='number')`
+    #  bit?
+    # |_https://jcristharif.com/msgspec/supported-types.html#decimal
+    # -[ ] should we also use it for .size?
+    #
+    price: Decimal
     size: float  # -ve is "sell", +ve is "buy"
 
     brokers: list[str] = []
@@ -178,7 +187,7 @@ class BrokerdOrder(Struct):
     time_ns: int
 
     symbol: str  # fqme
-    price: float
+    price: Decimal
     size: float
 
     # TODO: if we instead rely on a +ve/-ve size to determine

piker/clearing/_paper_engine.py

@@ -510,7 +510,7 @@ async def handle_order_requests(
                 reqid = await client.submit_limit(
                     oid=order.oid,
                     symbol=f'{order.symbol}.{client.broker}',
-                    price=order.price,
+                    price=float(order.price),
                     action=order.action,
                     size=order.size,
                     # XXX: by default 0 tells ``ib_insync`` methods that
@@ -655,6 +655,7 @@ async def open_trade_dialog(
                 # in) use manually constructed table from calling
                 # the `.get_mkt_info()` provider EP above.
                 _mktmap_table=mkt_by_fqme,
+                only_require=list(mkt_by_fqme),
             )
 
             pp_msgs: list[BrokerdPosition] = []

piker/cli/__init__.py

@@ -134,8 +134,8 @@ def pikerd(
     Spawn the piker broker-daemon.
 
     '''
-    from tractor.devx import maybe_open_crash_handler
+    # from tractor.devx import maybe_open_crash_handler
-    with maybe_open_crash_handler(pdb=pdb):
+    # with maybe_open_crash_handler(pdb=False):
     log = get_console_log(loglevel, name='cli')
 
     if pdb:
@@ -178,39 +178,18 @@ def pikerd(
 
     async def main():
         service_mngr: service.Services
-
         async with (
             service.open_pikerd(
                 registry_addrs=regaddrs,
                 loglevel=loglevel,
                 debug_mode=pdb,
-
+                # enable_transports=['uds'],
-                ) as service_mngr,  # normally delivers a ``Services`` handle
+                enable_transports=['tcp'],
-
+            ) as service_mngr,
-                # AsyncExitStack() as stack,
         ):
-                # TODO: spawn all other sub-actor daemons according to
-                # multiaddress endpoint spec defined by user config
             assert service_mngr
-
+            # ?TODO? spawn all other sub-actor daemons according to
-                # if tsdb:
+            # multiaddress endpoint spec defined by user config
-                #     dname, conf = await stack.enter_async_context(
-                #         service.marketstore.start_ahab_daemon(
-                #             service_mngr,
-                #             loglevel=loglevel,
-                #         )
-                #     )
-                #     log.info(f'TSDB `{dname}` up with conf:\n{conf}')
-
-                # if es:
-                #     dname, conf = await stack.enter_async_context(
-                #         service.elastic.start_ahab_daemon(
-                #             service_mngr,
-                #             loglevel=loglevel,
-                #         )
-                #     )
-                #     log.info(f'DB `{dname}` up with conf:\n{conf}')
-
             await trio.sleep_forever()
 
     trio.run(main)
@@ -328,6 +307,10 @@ def services(config, tl, ports):
     if not ports:
         ports = [_default_registry_port]
 
+    addr = tractor._addr.wrap_address(
+        addr=(host, ports[0])
+    )
+
     async def list_services():
         nonlocal host
         async with (
@@ -335,16 +318,18 @@ def services(config, tl, ports):
                 name='service_query',
                 loglevel=config['loglevel'] if tl else None,
             ),
-            tractor.get_arbiter(
+            tractor.get_registry(
-                host=host,
+                addr=addr,
-                port=ports[0]
             ) as portal
         ):
-            registry = await portal.run_from_ns('self', 'get_registry')
+            registry = await portal.run_from_ns(
+                'self',
+                'get_registry',
+            )
             json_d = {}
             for key, socket in registry.items():
-                host, port = socket
+                json_d[key] = f'{socket}'
-                json_d[key] = f'{host}:{port}'
+
             click.echo(f"{colorize_json(json_d)}")
 
     trio.run(list_services)

piker/config.py

@@ -41,10 +41,13 @@ from .log import get_logger
 log = get_logger('broker-config')
 
 
-# XXX NOTE: taken from ``click`` since apparently they have some
+# XXX NOTE: taken from `click`
-# super weirdness with sigint and sudo..no clue
+# |_https://github.com/pallets/click/blob/main/src/click/utils.py#L449
-# we're probably going to slowly just modify it to our own version over
+#
-# time..
+# (since apparently they have some super weirdness with SIGINT and
+# sudo.. no clue we're probably going to slowly just modify it to our
+# own version over time..)
+#
 def get_app_dir(
     app_name: str,
     roaming: bool = True,
@@ -261,7 +264,7 @@ def load(
         MutableMapping,
     ] = tomllib.loads,
 
-    touch_if_dne: bool = False,
+    touch_if_dne: bool = True,
 
     **tomlkws,
 
@@ -270,7 +273,7 @@ def load(
     Load config file by name.
 
     If desired config is not in the top level piker-user config path then
-    pass the ``path: Path`` explicitly.
+    pass the `path: Path` explicitly.
 
     '''
     # create the $HOME/.config/piker dir if dne
@@ -285,7 +288,8 @@ def load(
 
     if (
         not path.is_file()
-        and touch_if_dne
+        and
+        touch_if_dne
     ):
         # only do a template if no path provided,
         # just touch an empty file with same name.

piker/data/_sampling.py

@@ -740,7 +740,7 @@ async def sample_and_broadcast(
 
                                 log.warning(
                                     f'Feed OVERRUN {sub_key}'
-                                    '@{bus.brokername} -> \n'
+                                    f'@{bus.brokername} -> \n'
                                     f'feed @ {chan.uid}\n'
                                     f'throttle = {throttle} Hz'
                                 )

piker/data/_symcache.py

@@ -31,6 +31,7 @@ from pathlib import Path
 from pprint import pformat
 from typing import (
     Any,
+    Callable,
     Sequence,
     Hashable,
     TYPE_CHECKING,
@@ -56,7 +57,7 @@ from piker.brokers import (
 )
 
 if TYPE_CHECKING:
-    from ..accounting import (
+    from piker.accounting import (
         Asset,
         MktPair,
     )
@@ -90,6 +91,18 @@ class SymbologyCache(Struct):
     # provided by the backend pkg.
     mktmaps: dict[str, MktPair] = field(default_factory=dict)
 
+    def pformat(self) -> str:
+        return (
+            f'<{type(self).__name__}(\n'
+            f' .mod: {self.mod!r}\n'
+            f' .assets: {len(self.assets)!r}\n'
+            f' .pairs: {len(self.pairs)!r}\n'
+            f' .mktmaps: {len(self.mktmaps)!r}\n'
+            f')>'
+        )
+
+    __repr__ = pformat
+
     def write_config(self) -> None:
 
         # put the backend's pair-struct type ref at the top
@@ -149,19 +162,36 @@ class SymbologyCache(Struct):
                     'Implement `Client.get_assets()`!'
                 )
 
-            if get_mkt_pairs := getattr(client, 'get_mkt_pairs', None):
+            get_mkt_pairs: Callable|None = getattr(
+                client,
+                'get_mkt_pairs',
+                None,
+            )
+            if not get_mkt_pairs:
+                log.warning(
+                    'No symbology cache `Pair` support for `{provider}`..\n'
+                    'Implement `Client.get_mkt_pairs()`!'
+                )
+                return self
 
             pairs: dict[str, Struct] = await get_mkt_pairs()
-                for bs_fqme, pair in pairs.items():
+            if not pairs:
+                log.warning(
+                    'No pairs from intial {provider!r} sym-cache request?\n\n'
+                    '`Client.get_mkt_pairs()` -> {pairs!r} ?'
+                )
+                return self
 
-                    # NOTE: every backend defined pair should
+            for bs_fqme, pair in pairs.items():
-                    # declare it's ns path for roundtrip
-                    # serialization lookup.
                 if not getattr(pair, 'ns_path', None):
+                    # XXX: every backend defined pair must declare
+                    # a `.ns_path: tractor.NamespacePath` to enable
+                    # roundtrip serialization lookup from a local
+                    # cache file.
                     raise TypeError(
                         f'Pair-struct for {self.mod.name} MUST define a '
-                            '`.ns_path: str`!\n'
+                        '`.ns_path: str`!\n\n'
-                            f'{pair}'
+                        f'{pair!r}'
                     )
 
                 entry = await self.mod.get_mkt_info(pair.bs_fqme)
@@ -195,12 +225,6 @@ class SymbologyCache(Struct):
                 pair,
             )
 
-            else:
-                log.warning(
-                    'No symbology cache `Pair` support for `{provider}`..\n'
-                    'Implement `Client.get_mkt_pairs()`!'
-                )
-
         return self
 
     @classmethod

piker/data/_web_bs.py

@@ -27,7 +27,6 @@ from functools import partial
 from types import ModuleType
 from typing import (
     Any,
-    Optional,
     Callable,
     AsyncContextManager,
     AsyncGenerator,
@@ -35,6 +34,7 @@ from typing import (
 )
 import json
 
+import tractor
 import trio
 from trio_typing import TaskStatus
 from trio_websocket import (
@@ -167,7 +167,7 @@ async def _reconnect_forever(
 
     async def proxy_msgs(
         ws: WebSocketConnection,
-        pcs: trio.CancelScope,  # parent cancel scope
+        rent_cs: trio.CancelScope,  # parent cancel scope
     ):
         '''
         Receive (under `timeout` deadline) all msgs from from underlying
@@ -192,7 +192,7 @@ async def _reconnect_forever(
                         f'{url} connection bail with:'
                     )
                     await trio.sleep(0.5)
-                    pcs.cancel()
+                    rent_cs.cancel()
 
                     # go back to reonnect loop in parent task
                     return
@@ -204,7 +204,7 @@ async def _reconnect_forever(
                         f'{src_mod}\n'
                         'WS feed seems down and slow af.. reconnecting\n'
                     )
-                    pcs.cancel()
+                    rent_cs.cancel()
 
                     # go back to reonnect loop in parent task
                     return
@@ -228,7 +228,12 @@ async def _reconnect_forever(
     nobsws._connected = trio.Event()
     task_status.started()
 
-    while not snd._closed:
+    mc_state: trio._channel.MemoryChannelState = snd._state
+    while (
+        mc_state.open_receive_channels > 0
+        and
+        mc_state.open_send_channels > 0
+    ):
         log.info(
             f'{src_mod}\n'
             f'{url} trying (RE)CONNECT'
@@ -237,10 +242,11 @@ async def _reconnect_forever(
         ws: WebSocketConnection
         try:
             async with (
-                trio.open_nursery() as n,
                 open_websocket_url(url) as ws,
+                tractor.trionics.collapse_eg(),
+                trio.open_nursery() as tn,
             ):
-                cs = nobsws._cs = n.cancel_scope
+                cs = nobsws._cs = tn.cancel_scope
                 nobsws._ws = ws
                 log.info(
                     f'{src_mod}\n'
@@ -248,7 +254,7 @@ async def _reconnect_forever(
                 )
 
                 # begin relay loop to forward msgs
-                n.start_soon(
+                tn.start_soon(
                     proxy_msgs,
                     ws,
                     cs,
@@ -262,7 +268,7 @@ async def _reconnect_forever(
 
                     # TODO: should we return an explicit sub-cs
                     # from this fixture task?
-                    await n.start(
+                    await tn.start(
                         open_fixture,
                         fixture,
                         nobsws,
@@ -272,11 +278,23 @@ async def _reconnect_forever(
                 # to let tasks run **inside** the ws open block above.
                 nobsws._connected.set()
                 await trio.sleep_forever()
-        except HandshakeError:
+
+        except (
+            HandshakeError,
+            ConnectionRejected,
+        ):
             log.exception('Retrying connection')
+            await trio.sleep(0.5)  # throttle
 
-        # ws & nursery block ends
+        except BaseException as _berr:
+            berr = _berr
+            log.exception(
+                'Reconnect-attempt failed ??\n'
+            )
+            await trio.sleep(0.2)  # throttle
+            raise berr
 
+        #|_ws & nursery block ends
         nobsws._connected = trio.Event()
         if cs.cancelled_caught:
             log.cancel(
@@ -324,21 +342,25 @@ async def open_autorecon_ws(
     connetivity errors, or some user defined recv timeout.
 
     You can provide a ``fixture`` async-context-manager which will be
-    entered/exitted around each connection reset; eg. for (re)requesting
+    entered/exitted around each connection reset; eg. for
-    subscriptions without requiring streaming setup code to rerun.
+    (re)requesting subscriptions without requiring streaming setup
+    code to rerun.
 
     '''
     snd: trio.MemorySendChannel
     rcv: trio.MemoryReceiveChannel
     snd, rcv = trio.open_memory_channel(616)
 
-    async with trio.open_nursery() as n:
+    async with (
+        tractor.trionics.collapse_eg(),
+        trio.open_nursery() as tn
+    ):
         nobsws = NoBsWs(
             url,
             rcv,
             msg_recv_timeout=msg_recv_timeout,
         )
-        await n.start(
+        await tn.start(
             partial(
                 _reconnect_forever,
                 url,
@@ -351,11 +373,10 @@ async def open_autorecon_ws(
         await nobsws._connected.wait()
         assert nobsws._cs
         assert nobsws.connected()
-
         try:
             yield nobsws
         finally:
-            n.cancel_scope.cancel()
+            tn.cancel_scope.cancel()
 
 
 '''
@@ -368,8 +389,8 @@ of msgs over a `NoBsWs`.
 class JSONRPCResult(Struct):
     id: int
     jsonrpc: str = '2.0'
-    result: Optional[dict] = None
+    result: dict|None = None
-    error: Optional[dict] = None
+    error: dict|None = None
 
 
 @acm

piker/data/feed.py

@@ -39,6 +39,7 @@ from typing import (
     AsyncContextManager,
     Awaitable,
     Sequence,
+    TYPE_CHECKING,
 )
 
 import trio
@@ -75,6 +76,10 @@ from ._sampling import (
     uniform_rate_send,
 )
 
+if TYPE_CHECKING:
+    from tractor._addr import Address
+    from tractor.msg.types import Aid
+
 
 class Sub(Struct, frozen=True):
     '''
@@ -352,7 +357,9 @@ async def allocate_persistent_feed(
 
     # yield back control to starting nursery once we receive either
     # some history or a real-time quote.
-    log.info(f'loading OHLCV history: {fqme}')
+    log.info(
+        f'loading OHLCV history: {fqme!r}\n'
+    )
     await some_data_ready.wait()
 
     flume = Flume(
@@ -723,7 +730,10 @@ class Feed(Struct):
                 async for msg in stream:
                     await tx.send(msg)
 
-        async with trio.open_nursery() as nurse:
+        async with (
+            tractor.trionics.collapse_eg(),
+            trio.open_nursery() as nurse
+        ):
             # spawn a relay task for each stream so that they all
             # multiplex to a common channel.
             for brokername in mods:
@@ -786,7 +796,6 @@ async def install_brokerd_search(
 
 @acm
 async def maybe_open_feed(
-
     fqmes: list[str],
     loglevel: str | None = None,
 
@@ -840,7 +849,6 @@ async def maybe_open_feed(
 
 @acm
 async def open_feed(
-
     fqmes: list[str],
 
     loglevel: str|None = None,
@@ -899,19 +907,19 @@ async def open_feed(
             feed.portals[brokermod] = portal
 
             # fill out "status info" that the UI can show
-            host, port = portal.channel.raddr
+            chan: tractor.Channel = portal.chan
-            if host == '127.0.0.1':
+            raddr: Address = chan.raddr
-                host = 'localhost'
+            aid: Aid = chan.aid
-
+            # TAG_feed_status_update
             feed.status.update({
-                'actor_name': portal.channel.uid[0],
+                'actor_id': aid,
-                'host': host,
+                'actor_short_id': f'{aid.name}@{aid.pid}',
-                'port': port,
+                'ipc': chan.raddr.proto_key,
+                'ipc_addr': raddr,
                 'hist_shm': 'NA',
                 'rt_shm': 'NA',
-                'throttle_rate': tick_throttle,
+                'throttle_hz': tick_throttle,
             })
-            # feed.status.update(init_msg.pop('status', {}))
 
             # (allocate and) connect to any feed bus for this broker
             bus_ctxs.append(

piker/data/flows.py

@@ -36,10 +36,10 @@ from ._sharedmem import (
     ShmArray,
     _Token,
 )
+from piker.accounting import MktPair
 
 if TYPE_CHECKING:
-    from ..accounting import MktPair
+    from piker.data.feed import Feed
-    from .feed import Feed
 
 
 class Flume(Struct):

piker/data/validate.py

@@ -113,9 +113,9 @@ def validate_backend(
             )
             if ep is None:
                 log.warning(
-                    f'Provider backend {mod.name} is missing '
+                    f'Provider backend {mod.name!r} is missing '
-                    f'{daemon_name} support :(\n'
+                    f'{daemon_name!r} support?\n'
-                    f'The following endpoint is missing: {name}'
+                    f'|_module endpoint-func missing: {name!r}\n'
                 )
 
     inits: list[

piker/fsp/_engine.py

@@ -498,6 +498,7 @@ async def cascade(
 
         func_name: str = func.__name__
         async with (
+            tractor.trionics.collapse_eg(),  # avoid multi-taskc tb in console
             trio.open_nursery() as tn,
         ):
             # TODO: might be better to just make a "restart" method where

piker/service/_actor_runtime.py

@@ -107,17 +107,22 @@ async def open_piker_runtime(
         async with (
             tractor.open_root_actor(
 
-                # passed through to ``open_root_actor``
+                # passed through to `open_root_actor`
                 registry_addrs=registry_addrs,
                 name=name,
+                start_method=start_method,
                 loglevel=loglevel,
                 debug_mode=debug_mode,
-                start_method=start_method,
+
+                # XXX NOTE MEMBER DAT der's a perf hit yo!!
+                # https://greenback.readthedocs.io/en/latest/principle.html#performance
+                maybe_enable_greenback=True,
 
                 # TODO: eventually we should be able to avoid
                 # having the root have more then permissions to
                 # spawn other specialized daemons I think?
                 enable_modules=enable_modules,
+                hide_tb=False,
 
                 **tractor_kwargs,
             ) as actor,
@@ -200,7 +205,8 @@ async def open_pikerd(
             reg_addrs,
         ),
         tractor.open_nursery() as actor_nursery,
-        trio.open_nursery() as service_nursery,
+        tractor.trionics.collapse_eg(),
+        trio.open_nursery() as service_tn,
     ):
         for addr in reg_addrs:
             if addr not in root_actor.accept_addrs:
@@ -211,7 +217,7 @@ async def open_pikerd(
 
         # assign globally for future daemon/task creation
         Services.actor_n = actor_nursery
-        Services.service_n = service_nursery
+        Services.service_n = service_tn
         Services.debug_mode = debug_mode
 
         try:
@@ -221,7 +227,7 @@ async def open_pikerd(
             # TODO: is this more clever/efficient?
             # if 'samplerd' in Services.service_tasks:
             #     await Services.cancel_service('samplerd')
-            service_nursery.cancel_scope.cancel()
+            service_tn.cancel_scope.cancel()
 
 
 # TODO: do we even need this?
@@ -256,7 +262,10 @@ async def maybe_open_pikerd(
     loglevel: str | None = None,
     **kwargs,
 
-) -> tractor._portal.Portal | ClassVar[Services]:
+) -> (
+    tractor._portal.Portal
+    |ClassVar[Services]
+):
     '''
     If no ``pikerd`` daemon-root-actor can be found start it and
     yield up (we should probably figure out returning a portal to self
@@ -281,7 +290,8 @@ async def maybe_open_pikerd(
 
     registry_addrs: list[tuple[str, int]] = (
         registry_addrs
-        or [_default_reg_addr]
+        or
+        [_default_reg_addr]
     )
 
     pikerd_portal: tractor.Portal|None

piker/service/_daemon.py

@@ -28,6 +28,7 @@ from contextlib import (
 )
 
 import tractor
+from trio.lowlevel import current_task
 
 from ._util import (
     log,  # sub-sys logger
@@ -70,6 +71,7 @@ async def maybe_spawn_daemon(
     lock = Services.locks[service_name]
     await lock.acquire()
 
+    try:
         async with find_service(
             service_name,
             registry_addrs=[('127.0.0.1', 6116)],
@@ -134,6 +136,20 @@ async def maybe_spawn_daemon(
                 yield portal
                 await portal.cancel_actor()
 
+    except BaseException as _err:
+        err = _err
+        if (
+            lock.locked()
+            and
+            lock.statistics().owner is current_task()
+        ):
+            log.exception(
+                f'Releasing stale lock after crash..?'
+                f'{err!r}\n'
+            )
+            lock.release()
+            raise err
+
 
 async def spawn_emsd(
 

piker/service/_mngr.py

@@ -109,7 +109,7 @@ class Services:
                         # wait on any context's return value
                         # and any final portal result from the
                         # sub-actor.
-                        ctx_res: Any = await ctx.result()
+                        ctx_res: Any = await ctx.wait_for_result()
 
                         # NOTE: blocks indefinitely until cancelled
                         # either by error from the target context

piker/service/_registry.py

@@ -101,13 +101,15 @@ async def open_registry(
 
     if (
         not tractor.is_root_process()
-        and not Registry.addrs
+        and
+        not Registry.addrs
     ):
         Registry.addrs.extend(actor.reg_addrs)
 
     if (
         ensure_exists
-        and not Registry.addrs
+        and
+        not Registry.addrs
     ):
         raise RuntimeError(
             f"`{uid}` registry should already exist but doesn't?"
@@ -146,7 +148,7 @@ async def find_service(
     | list[Portal]
     | None
 ):
-
+    # try:
     reg_addrs: list[tuple[str, int]]
     async with open_registry(
         addrs=(
@@ -157,22 +159,39 @@ async def find_service(
             or Registry.addrs
         ),
     ) as reg_addrs:
-        log.info(f'Scanning for service `{service_name}`')
 
-        maybe_portals: list[Portal] | Portal | None
+        log.info(
+            f'Scanning for service {service_name!r}'
+        )
 
         # attach to existing daemon by name if possible
+        maybe_portals: list[Portal]|Portal|None
         async with tractor.find_actor(
             service_name,
             registry_addrs=reg_addrs,
             only_first=first_only,  # if set only returns single ref
         ) as maybe_portals:
             if not maybe_portals:
+                # log.info(
+                print(
+                    f'Could NOT find service {service_name!r} -> {maybe_portals!r}'
+                )
                 yield None
                 return
 
+            # log.info(
+            print(
+                f'Found service {service_name!r} -> {maybe_portals}'
+            )
             yield maybe_portals
 
+    # except BaseException as _berr:
+    #     berr = _berr
+    #     log.exception(
+    #         'tractor.find_actor() failed with,\n'
+    #     )
+    #     raise berr
+
 
 async def check_for_service(
     service_name: str,

piker/tsp/__init__.py

@@ -963,7 +963,10 @@ async def tsdb_backfill(
         # concurrently load the provider's most-recent-frame AND any
         # pre-existing tsdb history already saved in `piker` storage.
         dt_eps: list[DateTime, DateTime] = []
-        async with trio.open_nursery() as tn:
+        async with (
+            tractor.trionics.collapse_eg(),
+            trio.open_nursery() as tn
+        ):
             tn.start_soon(
                 push_latest_frame,
                 dt_eps,
@@ -1012,9 +1015,16 @@ async def tsdb_backfill(
                 int,
                 Duration,
             ]|None = config.get('frame_types', None)
+
             if def_frame_durs:
                 def_frame_size: Duration = def_frame_durs[timeframe]
-                assert def_frame_size == calced_frame_size
+
+                if def_frame_size != calced_frame_size:
+                    log.warning(
+                        f'Expected frame size {def_frame_size}\n'
+                        f'Rxed frame {calced_frame_size}\n'
+                    )
+                    # await tractor.pause()
             else:
                 # use what we calced from first frame above.
                 def_frame_size = calced_frame_size
@@ -1043,7 +1053,9 @@ async def tsdb_backfill(
             # if there is a gap to backfill from the first
             # history frame until the last datum loaded from the tsdb
             # continue that now in the background
-            async with trio.open_nursery() as tn:
+            async with trio.open_nursery(
+                strict_exception_groups=False,
+            ) as tn:
 
                 bf_done = await tn.start(
                     partial(
@@ -1308,6 +1320,7 @@ async def manage_history(
         # sampling period) data set since normally differently
         # sampled timeseries can be loaded / process independently
         # ;)
+        tractor.trionics.collapse_eg(),
         trio.open_nursery() as tn,
     ):
         log.info(

piker/tsp/_anal.py

@@ -517,7 +517,7 @@ def with_dts(
 
     '''
     return df.with_columns([
-        pl.col(time_col).shift(1).suffix('_prev'),
+        pl.col(time_col).shift(1).name.suffix('_prev'),
         pl.col(time_col).diff().alias('s_diff'),
         pl.from_epoch(pl.col(time_col)).alias('dt'),
     ]).with_columns([
@@ -623,7 +623,7 @@ def detect_vlm_gaps(
 
 ) -> pl.DataFrame:
 
-    vnull: pl.DataFrame = w_dts.filter(
+    vnull: pl.DataFrame = df.filter(
         pl.col(col) == 0
     )
     return vnull

piker/ui/_app.py

@@ -21,6 +21,7 @@ Main app startup and run.
 from functools import partial
 from types import ModuleType
 
+import tractor
 import trio
 
 from piker.ui.qt import (
@@ -116,6 +117,7 @@ async def _async_main(
         needed_brokermods[brokername] = brokers[brokername]
 
     async with (
+        tractor.trionics.collapse_eg(),
         trio.open_nursery() as root_n,
     ):
         # set root nursery and task stack for spawning other charts/feeds

piker/ui/_chart.py

@@ -33,7 +33,6 @@ import trio
 
 from piker.ui.qt import (
     QtCore,
-    QtWidgets,
     Qt,
     QLineF,
     QFrame,

piker/ui/_display.py

@@ -1445,7 +1445,10 @@ async def display_symbol_data(
         # for pause/resume on mouse interaction
         rt_chart.feed = feed
 
-        async with trio.open_nursery() as ln:
+        async with (
+            tractor.trionics.collapse_eg(),
+            trio.open_nursery() as ln,
+        ):
             # if available load volume related built-in display(s)
             vlm_charts: dict[
                 str,

piker/ui/_event.py

@@ -22,7 +22,10 @@ from contextlib import asynccontextmanager as acm
 from typing import Callable
 
 import trio
-from tractor.trionics import gather_contexts
+from tractor.trionics import (
+    gather_contexts,
+    collapse_eg,
+)
 
 from piker.ui.qt import (
     QtCore,
@@ -207,7 +210,10 @@ async def open_signal_handler(
         async for args in recv:
             await async_handler(*args)
 
-    async with trio.open_nursery() as tn:
+    async with (
+        collapse_eg(),
+        trio.open_nursery() as tn
+    ):
         tn.start_soon(proxy_to_handler)
         async with send:
             yield
@@ -242,6 +248,7 @@ async def open_handlers(
     widget: QWidget
     streams: list[trio.abc.ReceiveChannel]
     async with (
+        collapse_eg(),
         trio.open_nursery() as tn,
         gather_contexts([
             open_event_stream(

piker/ui/_feedstatus.py

@@ -18,10 +18,11 @@
 Feed status and controls widget(s) for embedding in a UI-pane.
 
 """
-
 from __future__ import annotations
-from textwrap import dedent
+from typing import (
-from typing import TYPE_CHECKING
+    Any,
+    TYPE_CHECKING,
+)
 
 # from PyQt5.QtCore import Qt
 
@@ -49,35 +50,55 @@ def mk_feed_label(
     a feed control protocol.
 
     '''
-    status = feed.status
+    status: dict[str, Any] = feed.status
     assert status
 
-    msg = dedent("""
+    # SO tips on ws/nls,
-        actor: **{actor_name}**\n
+    # https://stackoverflow.com/a/15721400
-        |_ @**{host}:{port}**\n
+    ws: str = ' '
-    """)
+    # nl: str = '<br>'  # dun work?
+    actor_info_repr: str = (
+        f')> **{status["actor_short_id"]}**\n'
+        '\n'  # bc md?
+    )
 
-    for key, val in status.items():
+    # fields to select *IN* for display
-        if key in ('host', 'port', 'actor_name'):
+    # (see `.data.feed.open_feed()` status
-            continue
+    #  update -> TAG_feed_status_update)
-        msg += f'\n|_ {key}: **{{{key}}}**\n'
+    for key in [
+        'ipc',
+        'hist_shm',
+        'rt_shm',
+        'throttle_hz',
+    ]:
+        # NOTE, the 2nd key is filled via `.format()` updates.
+        actor_info_repr += (
+            f'\n'  # bc md?
+            f'{ws}|_{key}: **{{{key}}}**\n'
+        )
+        # ^TODO? formatting and content..
+        # -[ ] showing which fqme is "forward" on the
+        #    chart/fsp/order-mode?
+        #  '|_ flows: **{symbols}**\n'
+        #
+        # -[x] why isn't the indent working?
+        #  => markdown, now solved..
 
     feed_label = FormatLabel(
-        fmt_str=msg,
+        fmt_str=actor_info_repr,
-        # |_ streams: **{symbols}**\n
         font=_font.font,
         font_size=_font_small.px_size,
         font_color='default_lightest',
     )
 
+    # ?TODO, remove this?
     # form.vbox.setAlignment(feed_label, Qt.AlignBottom)
     # form.vbox.setAlignment(Qt.AlignBottom)
-    _ = chart.height() - (
+    # _ = chart.height() - (
-        form.height() +
+    #     form.height() +
-        form.fill_bar.height()
+    #     form.fill_bar.height()
-        # feed_label.height()
+    #     # feed_label.height()
-    )
+    # )
 
     feed_label.format(**feed.status)
-
     return feed_label

piker/ui/_fsp.py

@@ -600,6 +600,7 @@ async def open_fsp_admin(
             kwargs=kwargs,
         ) as (cache_hit, cluster_map),
 
+        tractor.trionics.collapse_eg(),
         trio.open_nursery() as tn,
     ):
         if cache_hit:
@@ -613,6 +614,8 @@ async def open_fsp_admin(
         )
         try:
             yield admin
+
+        # ??TODO, does this *need* to be inside a finally?
         finally:
             # terminate all tasks via signals
             for key, entry in admin._registry.items():

piker/ui/_label.py

@@ -285,18 +285,20 @@ class FormatLabel(QLabel):
         font_size: int,
         font_color: str,
 
+        use_md: bool = True,
+
         parent=None,
 
     ) -> None:
 
         super().__init__(parent)
 
-        # by default set the format string verbatim and expect user to
+        # by default set the format string verbatim and expect user
-        # call ``.format()`` later (presumably they'll notice the
+        # to call ``.format()`` later (presumably they'll notice the
         # unformatted content if ``fmt_str`` isn't meant to be
         # unformatted).
         self.fmt_str = fmt_str
-        self.setText(fmt_str)
+        # self.setText(fmt_str)  # ?TODO, why here?
 
         self.setStyleSheet(
             f"""QLabel {{
@@ -306,6 +308,7 @@ class FormatLabel(QLabel):
             """
         )
         self.setFont(_font.font)
+        if use_md:
             self.setTextFormat(
                 Qt.TextFormat.MarkdownText
             )
@@ -316,7 +319,10 @@ class FormatLabel(QLabel):
             size_policy.Expanding,
         )
         self.setAlignment(
-            Qt.AlignVCenter | Qt.AlignLeft
+            Qt.AlignLeft
+            |
+            Qt.AlignBottom
+            # Qt.AlignVCenter
         )
         self.setText(self.fmt_str)
 

piker/ui/_search.py

@@ -15,7 +15,8 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 """
-qompleterz: embeddable search and complete using trio, Qt and rapidfuzz.
+qompleterz: embeddable search and complete using trio, Qt and
+rapidfuzz.
 
 """
 
@@ -46,6 +47,7 @@ import time
 from pprint import pformat
 
 from rapidfuzz import process as fuzzy
+import tractor
 import trio
 from trio_typing import TaskStatus
 
@@ -53,7 +55,7 @@ from piker.ui.qt import (
     size_policy,
     align_flag,
     Qt,
-    QtCore,
+    # QtCore,
     QtWidgets,
     QModelIndex,
     QItemSelectionModel,
@@ -920,7 +922,10 @@ async def fill_results(
 
             # issue multi-provider fan-out search request and place
             # "searching.." statuses on outstanding results providers
-            async with trio.open_nursery() as n:
+            async with (
+                tractor.trionics.collapse_eg(),
+                trio.open_nursery() as tn
+            ):
 
                 for provider, (search, pause) in (
                     _searcher_cache.copy().items()
@@ -944,7 +949,7 @@ async def fill_results(
                                 status_field='-> searchin..',
                             )
 
-                        await n.start(
+                        await tn.start(
                             pack_matches,
                             view,
                             has_results,
@@ -1004,12 +1009,14 @@ async def handle_keyboard_input(
     view.set_font_size(searchbar.dpi_font.px_size)
     send, recv = trio.open_memory_channel(616)
 
-    async with trio.open_nursery() as n:
+    async with (
-
+        tractor.trionics.collapse_eg(),  # needed?
+        trio.open_nursery() as tn
+    ):
         # start a background multi-searcher task which receives
         # patterns relayed from this keyboard input handler and
         # async updates the completer view's results.
-        n.start_soon(
+        tn.start_soon(
             partial(
                 fill_results,
                 searchw,

piker/ui/_style.py

@@ -269,6 +269,8 @@ def hcolor(name: str) -> str:
 
         # default ohlc-bars/curve gray
         'bracket': '#666666',  # like the logo
+        'pikers': '#616161',  # a trader shade of..
+        'beast': '#161616',  # in the dark alone.
 
         # bluish
         'charcoal': '#36454F',

piker/ui/order_mode.py

@@ -21,6 +21,7 @@ Chart trading, the only way to scalp.
 from __future__ import annotations
 from contextlib import asynccontextmanager
 from dataclasses import dataclass, field
+from decimal import Decimal
 from functools import partial
 from pprint import pformat
 import time
@@ -41,7 +42,6 @@ from piker.accounting import (
     Position,
     mk_allocator,
     MktPair,
-    Symbol,
 )
 from piker.clearing import (
     open_ems,
@@ -143,6 +143,15 @@ class OrderMode:
     }
     _staged_order: Order | None = None
 
+    @property
+    def curr_mkt(self) -> MktPair:
+        '''
+        Deliver the currently selected `MktPair` according
+        chart state.
+
+        '''
+        return self.chart.linked.mkt
+
     def on_level_change_update_next_order_info(
         self,
         level: float,
@@ -172,7 +181,11 @@ class OrderMode:
         line.update_labels(order_info)
 
         # update bound-in staged order
-        order.price = level
+        mkt: MktPair = self.curr_mkt
+        order.price: Decimal = mkt.quantize(
+            size=level,
+            quantity_type='price',
+        )
         order.size = order_info['size']
 
         # when an order is changed we flip the settings side-pane to
@@ -187,7 +200,9 @@ class OrderMode:
 
     ) -> LevelLine:
 
-        level = order.price
+        # TODO, if we instead just always decimalize at the ems layer
+        # we can avoid this back-n-forth casting?
+        level = float(order.price)
 
         line = order_line(
             chart or self.chart,
@@ -224,7 +239,11 @@ class OrderMode:
             # the order mode allocator but we still need to update the
             # "staged" order message we'll send to the ems
             def update_order_price(y: float) -> None:
-                order.price = y
+                mkt: MktPair = self.curr_mkt
+                order.price: Decimal = mkt.quantize(
+                    size=y,
+                    quantity_type='price',
+                )
 
             line._on_level_change = update_order_price
 
@@ -275,34 +294,31 @@ class OrderMode:
         chart = cursor.linked.chart
         if (
             not chart
-            and cursor
+            and
-            and cursor.active_plot
+            cursor
+            and
+            cursor.active_plot
         ):
             return
 
         chart = cursor.active_plot
-        price = cursor._datum_xy[1]
+        price: float = cursor._datum_xy[1]
         if not price:
             # zero prices are not supported by any means
             # since that's illogical / a no-op.
             return
 
-        mkt: MktPair = self.chart.linked.mkt
-
-        # NOTE : we could also use instead,
-        # mkt.quantize(price, quantity_type='price')
-        # but it returns a Decimal and it's probably gonna
-        # be slower?
         # TODO: should we be enforcing this precision
-        # at a different layer in the stack? right now
+        # at a different layer in the stack?
-        # any precision error will literally be relayed
+        # |_ might require `MktPair` tracking in the EMS?
-        # all the way back from the backend.
+        # |_ right now any precision error will be relayed
-
+        #    all the way back from the backend and vice-versa..
-        price = round(
+        #
-            price,
+        mkt: MktPair = self.curr_mkt
-            ndigits=mkt.price_tick_digits,
+        price: Decimal = mkt.quantize(
+            size=price,
+            quantity_type='price',
         )
-
         order = self._staged_order = Order(
             action=action,
             price=price,
@@ -378,7 +394,7 @@ class OrderMode:
                 'oid': oid,
             })
 
-        if order.price <= 0:
+        if float(order.price) <= 0:
             log.error(
                 '*!? Invalid `Order.price <= 0` ?!*\n'
                 # TODO: make this present multi-line in object form
@@ -515,14 +531,15 @@ class OrderMode:
             # if an order msg is provided update the line
             # **from** that msg.
             if order:
-                if order.price <= 0:
+                price: float = float(order.price)
+                if price <= 0:
                     log.error(f'Order has 0 price, cancelling..\n{order}')
                     self.cancel_orders([order.oid])
                     return None
 
-                line.set_level(order.price)
+                line.set_level(price)
                 self.on_level_change_update_next_order_info(
-                    level=order.price,
+                    level=price,
                     line=line,
                     order=order,
                     # use the corresponding position tracker for the
@@ -681,9 +698,9 @@ class OrderMode:
     ) -> Dialog | None:
         # NOTE: the `.order` attr **must** be set with the
         # equivalent order msg in order to be loaded.
-        order = msg.req
+        order: Order = msg.req
         oid = str(msg.oid)
-        symbol = order.symbol
+        symbol: str = order.symbol
 
         # TODO: MEGA UGGG ZONEEEE!
         src = msg.src
@@ -702,13 +719,22 @@ class OrderMode:
         order.oid = str(order.oid)
         order.brokers = [brokername]
 
-        # TODO: change this over to `MktPair`, but it's
+        # ?TODO? change this over to `MktPair`, but it's gonna be
-        # gonna be tough since we don't have any such data
+        # tough since we don't have any such data really in our
-        # really in our clearing msg schema..
+        # clearing msg schema..
-        order.symbol = Symbol.from_fqme(
+        # BUT WAIT! WHY do we even want/need this!?
-            fqsn=fqme,
+        #
-            info={},
+        # order.symbol = self.curr_mkt
-        )
+        #
+        # XXX, the old approach.. which i don't quire member why..
+        # -[ ] verify we for sure don't require this any more!
+        #  |_https://github.com/pikers/piker/issues/517
+        #
+        # order.symbol = Symbol.from_fqme(
+        #     fqsn=fqme,
+        #     info={},
+        # )
+
         maybe_dialog: Dialog | None = self.submit_order(
             send_msg=False,
             order=order,
@@ -766,6 +792,7 @@ async def open_order_mode(
             brokerd_accounts,
             ems_dialog_msgs,
         ),
+        tractor.trionics.collapse_eg(),
         trio.open_nursery() as tn,
 
     ):
@@ -1101,7 +1128,7 @@ async def process_trade_msg(
                         )
                     )
                 ):
-                    msg.req = order
+                    msg.req: Order = order
                     dialog: (
                         Dialog
                         # NOTE: on an invalid order submission (eg.
@@ -1166,7 +1193,7 @@ async def process_trade_msg(
             tm = time.time()
             mode.on_fill(
                 oid,
-                price=req.price,
+                price=float(req.price),
                 time_s=tm,
             )
             mode.lines.remove_line(uuid=oid)
@@ -1221,7 +1248,7 @@ async def process_trade_msg(
             tm = details['broker_time']
             mode.on_fill(
                 oid,
-                price=details['price'],
+                price=float(details['price']),
                 time_s=tm,
                 pointing='up' if action == 'buy' else 'down',
             )

snippets/qt_screen_info.py

@@ -1,4 +1,22 @@
-"""
+# piker: trading gear for hackers
+# Copyright (C) Tyler Goodlet (in stewardship for pikers)
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+'''
+A per-display, DPI (scaling) info dumper.
+
 Resource list for mucking with DPIs on multiple screens:
 
 - https://stackoverflow.com/questions/42141354/convert-pixel-size-to-point-size-for-fonts-on-multiple-platforms
@@ -12,89 +30,86 @@ Resource list for mucking with DPIs on multiple screens:
 - https://stackoverflow.com/questions/16561879/what-is-the-difference-between-logicaldpix-and-physicaldpix-in-qt
 - https://doc.qt.io/qt-5/qguiapplication.html#screenAt
 
-"""
+'''
 
 from pyqtgraph import QtGui
-from PyQt5.QtCore import (
+from PyQt6 import (
-     Qt, QCoreApplication
+    QtCore,
+    QtWidgets,
+)
+from PyQt6.QtCore import (
+    Qt,
+    QCoreApplication,
+    QSize,
+    QRect,
 )
 
 # Proper high DPI scaling is available in Qt >= 5.6.0. This attibute
 # must be set before creating the application
 if hasattr(Qt, 'AA_EnableHighDpiScaling'):
-    QCoreApplication.setAttribute(Qt.AA_EnableHighDpiScaling, True)
+    QCoreApplication.setAttribute(
+        Qt.AA_EnableHighDpiScaling,
+        True,
+    )
 
 if hasattr(Qt, 'AA_UseHighDpiPixmaps'):
-    QCoreApplication.setAttribute(Qt.AA_UseHighDpiPixmaps, True)
+    QCoreApplication.setAttribute(
+        Qt.AA_UseHighDpiPixmaps,
+        True,
+    )
 
-
+app = QtWidgets.QApplication([])
-app = QtGui.QApplication([])
+window = QtWidgets.QMainWindow()
-window = QtGui.QMainWindow()
+main_widget = QtWidgets.QWidget()
-main_widget = QtGui.QWidget()
 window.setCentralWidget(main_widget)
 window.show()
 
-pxr = main_widget.devicePixelRatioF()
+pxr: float = main_widget.devicePixelRatioF()
 
-# screen_num = app.desktop().screenNumber()
+# explicitly get main widget and primary displays
-# screen = app.screens()[screen_num]
+current_screen: QtGui.QScreen = app.screenAt(
+    main_widget.geometry().center()
+)
+primary_screen: QtGui.QScreen = app.primaryScreen()
 
-screen = app.screenAt(main_widget.geometry().center())
+screen: QtGui.QScreen
-
+for screen in app.screens():
-name = screen.name()
+    name: str = screen.name()
-size = screen.size()
+    model: str = screen.model().rstrip()
-geo = screen.availableGeometry()
+    size: QSize = screen.size()
-phydpi = screen.physicalDotsPerInch()
+    geo: QRect = screen.availableGeometry()
-logdpi = screen.logicalDotsPerInch()
+    phydpi: float = screen.physicalDotsPerInch()
+    logdpi: float = screen.logicalDotsPerInch()
+    is_primary: bool = screen is primary_screen
+    is_current: bool = screen is current_screen
 
     print(
-    # f'screen number: {screen_num}\n',
+        f'------ screen name: {name} ------\n'
-    f'screen name: {name}\n'
+        f'|_primary: {is_primary}\n'
-    f'screen size: {size}\n'
+        f' _current: {is_current}\n'
-    f'screen geometry: {geo}\n\n'
+        f' _model: {model}\n'
-    f'devicePixelRationF(): {pxr}\n'
+        f' _screen size: {size}\n'
-    f'physical dpi: {phydpi}\n'
+        f' _screen geometry: {geo}\n'
-    f'logical dpi: {logdpi}\n'
+        f' _devicePixelRationF(): {pxr}\n'
+        f' _physical dpi: {phydpi}\n'
+        f' _logical dpi: {logdpi}\n'
     )
 
-print('-'*50)
+# app-wide font info
-
-screen = app.primaryScreen()
-
-name = screen.name()
-size = screen.size()
-geo = screen.availableGeometry()
-phydpi = screen.physicalDotsPerInch()
-logdpi = screen.logicalDotsPerInch()
-
-print(
-    # f'screen number: {screen_num}\n',
-    f'screen name: {name}\n'
-    f'screen size: {size}\n'
-    f'screen geometry: {geo}\n\n'
-    f'devicePixelRationF(): {pxr}\n'
-    f'physical dpi: {phydpi}\n'
-    f'logical dpi: {logdpi}\n'
-)
-
-
-# app-wide font
 font = QtGui.QFont("Hack")
 # use pixel size to be cross-resolution compatible?
 font.setPixelSize(6)
 
-
 fm = QtGui.QFontMetrics(font)
-fontdpi = fm.fontDpi()
+fontdpi: float = fm.fontDpi()
-font_h = fm.height()
+font_h: int = fm.height()
-
-string = '10000'
-str_br = fm.boundingRect(string)
-str_w = str_br.width()
 
+string: str = '10000'
+str_br: QtCore.QRect = fm.boundingRect(string)
+str_w: int = str_br.width()
 
 print(
-    # f'screen number: {screen_num}\n',
+    f'------ global font settings ------\n'
     f'font dpi: {fontdpi}\n'
     f'font height: {font_h}\n'
     f'string bounding rect: {str_br}\n'

tags

@@ -0,0 +1 @@
+TAG_feed_status_update	./piker/data/feed.py	/TAG_feed_status_update/

tests/conftest.py

@@ -15,6 +15,12 @@ from piker.service import (
 from piker.log import get_console_log
 
 
+# include `tractor`'s built-in fixtures!
+pytest_plugins: tuple[str] = (
+    "tractor._testing.pytest",
+)
+
+
 def pytest_addoption(parser):
     parser.addoption("--ll", action="store", dest='loglevel',
                      default=None, help="logging level to set when testing")

tests/test_ems.py

@@ -179,7 +179,7 @@ def test_ems_err_on_bad_broker(
         # NOTE: emsd should error on the actor's enabled modules
         # import phase, when looking for a backend named `doggy`.
         except tractor.RemoteActorError as re:
-            assert re.type == ModuleNotFoundError
+            assert re.type is ModuleNotFoundError
 
     run_and_tollerate_cancels(load_bad_fqme)
 

tests/test_questrade.py

@@ -142,7 +142,12 @@ async def test_concurrent_tokens_refresh(us_symbols, loglevel):
         # async with tractor.open_nursery() as n:
         #     await n.run_in_actor('other', intermittently_refresh_tokens)
 
-        async with trio.open_nursery() as n:
+        async with (
+            tractor.trionics.collapse_eg(),
+            trio.open_nursery(
+            # strict_exception_groups=False,
+            ) as n
+        ):
 
             quoter = await qt.stock_quoter(client, us_symbols)
 
@@ -383,7 +388,9 @@ async def test_quote_streaming(tmx_symbols, loglevel, stream_what):
             else:
                 symbols = [tmx_symbols]
 
-            async with trio.open_nursery() as n:
+            async with trio.open_nursery(
+                strict_exception_groups=False,
+            ) as n:
                 for syms, func in zip(symbols, stream_what):
                     n.start_soon(func, feed, syms)