From 9c7ca84fef5665a629a4a94979d0231b407cecdc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 2 Dec 2018 00:37:27 -0500 Subject: [PATCH 01/42] Include strike and expiry in option quotes --- piker/brokers/questrade.py | 21 ++++++++++++++++++--- tests/test_questrade.py | 2 ++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 330580af..ea530c97 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -124,6 +124,7 @@ class Client: self._reload_config(config) self._symbol_cache: Dict[str, int] = {} self._contracts2expiries = {} + self._optids2contractinfo = {} def _reload_config(self, config=None, **kwargs): log.warn("Reloading access config data") @@ -323,6 +324,16 @@ class Client: item['strikePrice']: item for item in byroot['chainPerRoot'][0]['chainPerStrikePrice'] } + + # fill out contract id to strike expiry map + for key, bystrikes in by_key.items(): + for strike, ids in bystrikes.items(): + for elem in ('callSymbolId', 'putSymbolId'): + self._optids2contractinfo[ + ids[elem]] = { + 'strike': strike, + 'expiry': key.expiry, + } return by_key async def option_chains( @@ -339,6 +350,12 @@ class Client: # index by .symbol, .expiry since that's what # a subscriber (currently) sends initially quote['key'] = (key[0], key[2]) + # update with expiry and strike (Obviously the + # QT api designers are using some kind of severely + # stupid disparate table system where they keep + # contract info in a separate table from the quote format + # keys. I'm really not surprised though - windows shop..) + quote.update(self._optids2contractinfo[quote['symbolId']]) batch.extend(quotes) return batch @@ -471,9 +488,7 @@ async def option_quoter(client: Client, tickers: List[str]): if isinstance(tickers[0], tuple): datetime.fromisoformat(tickers[0][1]) else: - log.warn(f"Ignoring option quoter call with {tickers}") - # TODO make caller always check that a quoter has been set - return + raise ValueError(f'Option subscription format is (symbol, expiry)') @async_lifo_cache(maxsize=128) async def get_contract_by_date(sym_date_pairs: Tuple[Tuple[str, str]]): diff --git a/tests/test_questrade.py b/tests/test_questrade.py index 482c1e37..4c3ac059 100644 --- a/tests/test_questrade.py +++ b/tests/test_questrade.py @@ -53,6 +53,7 @@ _ex_quotes = { 'bidSize': 0, 'delay': 0, 'delta': -0.212857, + "expiry": "2021-01-15T00:00:00.000000-05:00", 'gamma': 0.003524, 'highPrice': 0, 'isHalted': False, @@ -66,6 +67,7 @@ _ex_quotes = { 'openInterest': 1, 'openPrice': 0, 'rho': -0.891868, + "strike": 8, 'symbol': 'WEED15Jan21P54.00.MX', 'symbolId': 22739148, 'theta': -0.012911, From fb47ea2e5a653ddcf18385da0232ae0a59511004 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 8 Dec 2018 18:40:08 -0500 Subject: [PATCH 02/42] Define option field structure Add some extra fields to each quote that QT should already be providing (instead of hiding them in the symbol and request contract info); namely, the expiry and contact type (i.e. put or call). Define the base set of fields to be displayed in an option chain UI and add a quote formatter. --- piker/brokers/questrade.py | 111 ++++++++++++++++++++++++++++++++++--- 1 file changed, 104 insertions(+), 7 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index ea530c97..d5d85514 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -326,13 +326,16 @@ class Client: } # fill out contract id to strike expiry map - for key, bystrikes in by_key.items(): + for tup, bystrikes in by_key.items(): for strike, ids in bystrikes.items(): - for elem in ('callSymbolId', 'putSymbolId'): + for key, contract_type in ( + ('callSymbolId', 'call'), ('putSymbolId', 'put') + ): self._optids2contractinfo[ - ids[elem]] = { + ids[key]] = { 'strike': strike, - 'expiry': key.expiry, + 'expiry': tup.expiry, + 'contract_type': contract_type, } return by_key @@ -571,15 +574,15 @@ _qt_stock_keys = { # BidAskLayout columns which will contain three cells the first stacked on top # of the other 2 -_bidasks = { +_stock_bidasks = { 'last': ['bid', 'ask'], 'size': ['bsize', 'asize'], 'VWAP': ['low', 'high'], - 'mktcap': ['vol', '$ vol'], + 'vol': ['mktcap', '$ vol'], } -def format_quote( +def format_stock_quote( quote: dict, symbol_data: dict, keymap: dict = _qt_stock_keys, @@ -624,3 +627,97 @@ def format_quote( displayable[new_key] = display_value return new, displayable + + +_qt_option_keys = { + "lastTradePrice": 'last', + "askPrice": 'ask', + "bidPrice": 'bid', + "lastTradeSize": 'size', + "bidSize": 'bsize', + "askSize": 'asize', + "volume": 'vol', + "VWAP": 'VWAP', + "lowPrice": 'low', + "highPrice": 'high', + # "expiry": "expiry", + # "delay": 0, + "delta": 'delta', + "gamma": 'gamma', + "rho": 'rho', + "theta": 'theta', + "vega": 'vega', + "$ vol": '$ vol', + # "2021-01-15T00:00:00.000000-05:00", + # "isHalted": false, + # "key": [ + # "APHA.TO", + # "2021-01-15T00:00:00.000000-05:00" + # ], + # "lastTradePriceTrHrs": null, + # "lastTradeTick": 'tick', + "lastTradeTime": 'time', + "openInterest": 'open_interest', + "openPrice": 'open', + # "strike": 'strike', + # "symbol": "APHA15Jan21P8.00.MX", + # "symbolId": 23881868, + # "underlying": "APHA.TO", + # "underlyingId": 8297492, + "symbol": 'symbol', + "contract_type": 'contract_type', + "volatility": 'volatility', + "strike": 'strike', +} + +_option_bidasks = { + 'last': ['bid', 'ask'], + 'size': ['bsize', 'asize'], + 'VWAP': ['low', 'high'], + 'vol': ['open_interest', '$ vol'], +} + + +def format_option_quote( + quote: dict, + symbol_data: dict, + keymap: dict = _qt_option_keys, +) -> Tuple[dict, dict]: + """Remap a list of quote dicts ``quotes`` using the mapping of old keys + -> new keys ``keymap`` returning 2 dicts: one with raw data and the other + for display. + + Returns 2 dicts: first is the original values mapped by new keys, + and the second is the same but with all values converted to a + "display-friendly" string format. + """ + # TODO: need historical data.. + # (cause why would QT keep their quote structure consistent across + # assets..) + # previous = symbol_data[symbol]['prevDayClosePrice'] + # change = percent_change(previous, last) + computed = { + # why QT do you have to be an asshole shipping null values!!! + '$ vol': round((quote['VWAP'] or 0) * (quote['volume'] or 0), 3), + # '%': round(change, 3), + # 'close': previous, + } + new = {} + displayable = {} + + # structuring and normalization + for key, new_key in keymap.items(): + display_value = value = computed.get(key) or quote.get(key) + + # API servers can return `None` vals when markets are closed (weekend) + value = 0 if value is None else value + + # convert values to a displayble format using available formatting func + if isinstance(new_key, tuple): + new_key, func = new_key + display_value = func(value) if value else value + + new[new_key] = value + displayable[new_key] = display_value + + return new, displayable From 54261ecc4c87a7f9f3cee1794e616397286b3d86 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 10 Dec 2018 01:49:19 -0500 Subject: [PATCH 03/42] Refer to async exit stack via feed --- piker/brokers/data.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/piker/brokers/data.py b/piker/brokers/data.py index e35b0dec..b6e5ab6a 100644 --- a/piker/brokers/data.py +++ b/piker/brokers/data.py @@ -116,6 +116,7 @@ class DataFeed(typing.NamedTuple): """ mod: ModuleType client: object + exit_stack: contextlib.AsyncExitStack quoter_keys: List[str] = ['stock', 'option'] tasks: Dict[str, trio._core._run.Task] = dict.fromkeys( quoter_keys, False) @@ -271,8 +272,6 @@ async def get_cached_feed( ss = tractor.current_actor().statespace feeds = ss.setdefault('feeds', {'_lock': trio.Lock()}) lock = feeds['_lock'] - feed_stacks = ss.setdefault('feed_stacks', {}) - feed_stack = feed_stacks.setdefault(brokername, contextlib.AsyncExitStack()) async with lock: try: feed = feeds[brokername] @@ -281,11 +280,13 @@ async def get_cached_feed( except KeyError: log.info(f"Creating new client for broker {brokername}") brokermod = get_brokermod(brokername) - client = await feed_stack.enter_async_context( + exit_stack = contextlib.AsyncExitStack() + client = await exit_stack.enter_async_context( brokermod.get_client()) feed = DataFeed( mod=brokermod, client=client, + exit_stack=exit_stack, ) feeds[brokername] = feed return feed @@ -306,7 +307,6 @@ async def start_quote_stream( Since most brokers seems to support batch quote requests we limit to one task per process for now. """ - actor = tractor.current_actor() # set log level after fork get_console_log(actor.loglevel) @@ -376,7 +376,7 @@ async def start_quote_stream( # broker2symbolsubs.pop(broker, None) # destroy the API client - await feed_stack.aclose() + await feed.exit_stack.aclose() async def stream_to_file( From 20778b02b5138e0e864074a5f358d7220c6d0ab8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 10 Dec 2018 01:50:00 -0500 Subject: [PATCH 04/42] Format numerical option fields --- piker/brokers/questrade.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index d5d85514..c9f2cc9e 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -636,18 +636,18 @@ _qt_option_keys = { "lastTradeSize": 'size', "bidSize": 'bsize', "askSize": 'asize', - "volume": 'vol', - "VWAP": 'VWAP', + 'VWAP': ('VWAP', partial(round, ndigits=3)), "lowPrice": 'low', "highPrice": 'high', # "expiry": "expiry", # "delay": 0, - "delta": 'delta', - "gamma": 'gamma', - "rho": 'rho', - "theta": 'theta', - "vega": 'vega', - "$ vol": '$ vol', + "delta": ('delta', partial(round, ndigits=3)), + "gamma": ('gama', partial(round, ndigits=3)), + "rho": ('rho', partial(round, ndigits=3)), + "theta": ('theta', partial(round, ndigits=3)), + "vega": ('vega', partial(round, ndigits=3)), + '$ vol': ('$ vol', humanize), + 'volume': ('vol', humanize), # "2021-01-15T00:00:00.000000-05:00", # "isHalted": false, # "key": [ @@ -657,7 +657,7 @@ _qt_option_keys = { # "lastTradePriceTrHrs": null, # "lastTradeTick": 'tick', "lastTradeTime": 'time', - "openInterest": 'open_interest', + "openInterest": 'oi', "openPrice": 'open', # "strike": 'strike', # "symbol": "APHA15Jan21P8.00.MX", @@ -666,7 +666,7 @@ _qt_option_keys = { # "underlyingId": 8297492, "symbol": 'symbol', "contract_type": 'contract_type', - "volatility": 'volatility', + "volatility": ('volatility', partial(round, ndigits=3)), "strike": 'strike', } @@ -674,7 +674,7 @@ _option_bidasks = { 'last': ['bid', 'ask'], 'size': ['bsize', 'asize'], 'VWAP': ['low', 'high'], - 'vol': ['open_interest', '$ vol'], + 'vol': ['oi', '$ vol'], } From 8647216b750e0946843cf602aa62bcc71d248609 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 10 Dec 2018 01:51:49 -0500 Subject: [PATCH 05/42] Tabular kivy UI improvements `Row`: - `no_cell`: support a list of keys for which no cells will be created - allow passing in a `cell_type` at instantiation `TickerTable`: - keep track of rendered rows via a private `_rendered` set - don't create rows inside `append_row()` expect caller to do it - never render already rendered widgets in `render_rows()` Miscellaneous: - generalize `update_quotes()` to not be tied to specific quote fields and allow passing in a quote `formatter()` func - don't bother creating a nursery block until necessary in main - more commenting --- piker/ui/monitor.py | 254 ++++++++++++++++++++++++++------------------ 1 file changed, 151 insertions(+), 103 deletions(-) diff --git a/piker/ui/monitor.py b/piker/ui/monitor.py index d9568135..9d15b257 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/monitor.py @@ -7,7 +7,7 @@ Launch with ``piker monitor ``. """ from itertools import chain from types import ModuleType, AsyncGeneratorType -from typing import List +from typing import List, Callable, Dict import trio import tractor @@ -19,7 +19,6 @@ from kivy.lang import Builder from kivy import utils from kivy.app import async_runTouchApp from kivy.core.window import Window -from async_generator import aclosing from ..log import get_logger from .pager import PagerView @@ -126,10 +125,11 @@ _kv = (f''' # row higlighting on mouse over Color: rgba: {_i3_rgba} - RoundedRectangle: + # RoundedRectangle: + Rectangle: size: self.width, self.height if self.hovered else 1 pos: self.pos - radius: (10,) + # radius: (0,) @@ -148,9 +148,10 @@ class Cell(Button): ``key`` is the column name index value. """ - def __init__(self, key=None, **kwargs): + def __init__(self, key=None, is_header=False, **kwargs): super(Cell, self).__init__(**kwargs) self.key = key + self.is_header = is_header class HeaderCell(Cell): @@ -178,6 +179,8 @@ class HeaderCell(Cell): # sort and render the rows immediately self.row.table.render_rows(table.quote_cache) + # TODO: make this some kind of small geometry instead + # (maybe like how trading view does it). # allow highlighting of row headers for tracking elif self.is_header: if self.background_color == self.color: @@ -227,7 +230,7 @@ class BidAskLayout(StackLayout): self.row = None def get_cell(self, key): - return self._keys2cells[key] + return self._keys2cells.get(key) @property def row(self): @@ -246,13 +249,16 @@ class BidAskLayout(StackLayout): class Row(GridLayout, HoverBehavior): """A grid for displaying a row of ticker quote data. - - The row fields can be updated using the ``fields`` property which will in - turn adjust the text color of the values based on content changes. """ def __init__( - self, record, headers=(), bidasks=None, table=None, + self, + record, + headers=(), + no_cell=(), + bidasks=None, + table=None, is_header=False, + cell_type=None, **kwargs ): super(Row, self).__init__(cols=len(record), **kwargs) @@ -260,11 +266,13 @@ class Row(GridLayout, HoverBehavior): self._last_record = record self.table = table self.is_header = is_header + self._cell_type = cell_type + self.widget = self - # selection state - self.mouse_over = False - - # create `BidAskCells` first + # Create `BidAskCells` first. + # bid/ask cells are just 3 cells grouped in a + # ``BidAskLayout`` which just stacks the parent cell + # on top of 2 children. layouts = {} bidasks = bidasks or {} ba_cells = {} @@ -291,20 +299,20 @@ class Row(GridLayout, HoverBehavior): elif key in children_flat: # these cells have already been added to the `BidAskLayout` continue - else: + elif key not in no_cell: cell = self._append_cell(val, key, header=header) cell.key = key self._cell_widgets[key] = cell def get_cell(self, key): - return self._cell_widgets[key] + return self._cell_widgets.get(key) def _append_cell(self, text, key, header=False): if not len(self._cell_widgets) < self.cols: raise ValueError(f"Can not append more then {self.cols} cells") # header cells just have a different colour - celltype = HeaderCell if header else Cell + celltype = self._cell_type or (HeaderCell if header else Cell) cell = celltype(text=str(text), key=key) cell.is_header = header cell.row = self @@ -312,7 +320,8 @@ class Row(GridLayout, HoverBehavior): return cell def update(self, record, displayable): - """Update this row's cells with new values from a quote ``record``. + """Update this row's cells with new values from a quote + ``record``. Return all cells that changed in a ``dict``. """ @@ -331,10 +340,13 @@ class Row(GridLayout, HoverBehavior): color = gray cell = self.get_cell(key) - cell.text = str(displayable[key]) - cell.color = color - if color != gray: - cells[key] = cell + # some displayable fields might have specifically + # not had cells created as set in the `no_cell` attr + if cell is not None: + cell.text = str(displayable[key]) + cell.color = color + if color != gray: + cells[key] = cell self._last_record = record return cells @@ -359,40 +371,53 @@ class Row(GridLayout, HoverBehavior): class TickerTable(GridLayout): """A grid for displaying ticker quote records as a table. """ - def __init__(self, sort_key='%', quote_cache={}, **kwargs): + def __init__(self, sort_key='%', **kwargs): super(TickerTable, self).__init__(**kwargs) self.symbols2rows = {} self.sort_key = sort_key - self.quote_cache = quote_cache + self.quote_cache = {} self.row_filter = lambda item: item # for tracking last clicked column header cell self.last_clicked_col_cell = None self._last_row_toggle = 0 + self._rendered = set() - def append_row(self, record, bidasks=None): + def append_row(self, key, row): """Append a `Row` of `Cell` objects to this table. """ - row = Row(record, headers=('symbol',), bidasks=bidasks, table=self) # store ref to each row - self.symbols2rows[row._last_record['symbol']] = row + self.symbols2rows[key] = row self.add_widget(row) return row def render_rows( - self, pairs: {str: (dict, Row)}, sort_key: str = None, - row_filter=None, + self, + pairs: Dict[str, Row], + sort_key: str = None, + row_filter=None, ): """Sort and render all rows on the ticker grid from ``pairs``. """ self.clear_widgets() + self._rendered.clear() sort_key = sort_key or self.sort_key - for data, row in filter( + # TODO: intead of constantly re-rendering on every + # change do a binary search insert using ``bisect.insort()`` + for row in filter( row_filter or self.row_filter, reversed( - sorted(pairs.values(), key=lambda item: item[0][sort_key]) + sorted( + pairs.values(), + key=lambda row: row._last_record[sort_key] + ) ) ): - self.add_widget(row) # row append + widget = row.widget + if widget not in self._rendered: + self.add_widget(widget) # row append + self._rendered.add(widget) + else: + log.debug(f"Skipping adding widget {widget}") def ticker_search(self, patt): """Return sequence of matches when pattern ``patt`` is in a @@ -402,6 +427,9 @@ class TickerTable(GridLayout): if patt in symbol: yield symbol, row + def get_row(self, symbol: str) -> Row: + return self.symbols2rows[symbol] + def search(self, patt): """Search bar api compat. """ @@ -410,7 +438,7 @@ class TickerTable(GridLayout): async def update_quotes( nursery: trio._core._run.Nursery, - brokermod: ModuleType, + formatter: Callable, widgets: dict, agen: AsyncGeneratorType, symbol_data: dict, @@ -426,24 +454,27 @@ async def update_quotes( for cell in cells: cell.background_color = _black_rgba - def color_row(row, data, cells): + def color_row(row, record, cells): hdrcell = row.get_cell('symbol') chngcell = row.get_cell('%') # determine daily change color - daychange = float(data['%']) - if daychange < 0.: - color = colorcode('red2') - elif daychange > 0.: - color = colorcode('forestgreen') - else: - color = colorcode('gray') + color = colorcode('gray') + percent_change = record.get('%') + if percent_change: + daychange = float(record['%']) + if daychange < 0.: + color = colorcode('red2') + elif daychange > 0.: + color = colorcode('forestgreen') # update row header and '%' cell text color - chngcell.color = hdrcell.color = color - # if the cell has been "highlighted" make sure to change its color - if hdrcell.background_color != [0]*4: - hdrcell.background_color = color + if chngcell: + chngcell.color = color + hdrcell.color = color + # if the cell has been "highlighted" make sure to change its color + if hdrcell.background_color != [0]*4: + hdrcell.background_color = color # briefly highlight bg of certain cells on each trade execution unflash = set() @@ -483,12 +514,12 @@ async def update_quotes( # initial coloring for sym, quote in first_quotes.items(): - row = table.symbols2rows[sym] - record, displayable = brokermod.format_quote( + row = table.get_row(sym) + record, displayable = formatter( quote, symbol_data=symbol_data) row.update(record, displayable) color_row(row, record, {}) - cache[sym] = (record, row) + cache[sym] = row # render all rows once up front table.render_rows(cache) @@ -496,12 +527,12 @@ async def update_quotes( # real-time cell update loop async for quotes in agen: # new quotes data only for symbol, quote in quotes.items(): - record, displayable = brokermod.format_quote( + record, displayable = formatter( quote, symbol_data=symbol_data) - row = table.symbols2rows[symbol] - cache[symbol] = (record, row) + row = table.get_row(symbol) cells = row.update(record, displayable) color_row(row, record, cells) + cache[symbol] = row table.render_rows(cache) log.debug("Waiting on quotes") @@ -528,6 +559,9 @@ async def _async_main( "piker.brokers.data", 'stream_from_file', filename=test ) + # TODO: need a set of test packets to make this work + # seriously fu QT + # sd = {} else: # start live streaming from broker daemon quote_gen = await portal.run( @@ -540,62 +574,69 @@ async def _async_main( "piker.brokers.data", 'symbol_data', broker=brokermod.name, tickers=tickers) + # get first quotes response + log.debug("Waiting on first quote...") + quotes = await quote_gen.__anext__() + first_quotes = [ + brokermod.format_stock_quote(quote, symbol_data=sd)[0] + for quote in quotes.values()] + + if first_quotes[0].get('last') is None: + log.error("Broker API is down temporarily") + return + + # build out UI + Window.set_title(f"monitor: {name}\t(press ? for help)") + Builder.load_string(_kv) + box = BoxLayout(orientation='vertical', spacing=0) + + # define bid-ask "stacked" cells + # (TODO: needs some rethinking and renaming for sure) + bidasks = brokermod._stock_bidasks + + # add header row + headers = first_quotes[0].keys() + header = Row( + {key: key for key in headers}, + headers=headers, + bidasks=bidasks, + is_header=True, + size_hint=(1, None), + ) + box.add_widget(header) + + # build table + table = TickerTable( + cols=1, + size_hint=(1, None), + ) + for ticker_record in first_quotes: + table.append_row( + ticker_record['symbol'], + Row(ticker_record, headers=('symbol',), bidasks=bidasks, table=table) + ) + + # associate the col headers row with the ticker table even though + # they're technically wrapped separately in containing BoxLayout + header.table = table + + # mark the initial sorted column header as bold and underlined + sort_cell = header.get_cell(table.sort_key) + sort_cell.bold = sort_cell.underline = True + table.last_clicked_col_cell = sort_cell + + # set up a pager view for large ticker lists + table.bind(minimum_height=table.setter('height')) + async with trio.open_nursery() as nursery: - # get first quotes response - log.debug("Waiting on first quote...") - quotes = await quote_gen.__anext__() - first_quotes = [ - brokermod.format_quote(quote, symbol_data=sd)[0] - for quote in quotes.values()] - - if first_quotes[0].get('last') is None: - log.error("Broker API is down temporarily") - nursery.cancel_scope.cancel() - return - - # build out UI - Window.set_title(f"monitor: {name}\t(press ? for help)") - Builder.load_string(_kv) - box = BoxLayout(orientation='vertical', spacing=0) - - # define bid-ask "stacked" cells - # (TODO: needs some rethinking and renaming for sure) - bidasks = brokermod._bidasks - - # add header row - headers = first_quotes[0].keys() - header = Row( - {key: key for key in headers}, - headers=headers, - bidasks=bidasks, - is_header=True, - size_hint=(1, None), + pager = PagerView( + container=box, + contained=table, + nursery=nursery ) - box.add_widget(header) - - # build table - table = TickerTable( - cols=1, - size_hint=(1, None), - ) - for ticker_record in first_quotes: - table.append_row(ticker_record, bidasks=bidasks) - # associate the col headers row with the ticker table even though - # they're technically wrapped separately in containing BoxLayout - header.table = table - - # mark the initial sorted column header as bold and underlined - sort_cell = header.get_cell(table.sort_key) - sort_cell.bold = sort_cell.underline = True - table.last_clicked_col_cell = sort_cell - - # set up a pager view for large ticker lists - table.bind(minimum_height=table.setter('height')) - pager = PagerView(box, table, nursery) box.add_widget(pager) widgets = { - # 'anchor': anchor, 'root': box, 'table': table, 'box': box, @@ -603,7 +644,14 @@ async def _async_main( 'pager': pager, } nursery.start_soon( - update_quotes, nursery, brokermod, widgets, quote_gen, sd, quotes) + update_quotes, + nursery, + brokermod.format_stock_quote, + widgets, + quote_gen, + sd, + quotes + ) try: # Trio-kivy entry point. From 201919eef710c64425f91b89ad45fadc06de320a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 10 Dec 2018 02:00:10 -0500 Subject: [PATCH 06/42] Initial option chain UI Spin it up with `piker optschain`. Still lots of polishing and features to add but it's a start! --- piker/cli.py | 43 +++++- piker/ui/option_chain.py | 275 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 315 insertions(+), 3 deletions(-) create mode 100644 piker/ui/option_chain.py diff --git a/piker/cli.py b/piker/cli.py index 6dd3b327..e282f669 100644 --- a/piker/cli.py +++ b/piker/cli.py @@ -22,6 +22,10 @@ DEFAULT_BROKER = 'robinhood' _config_dir = click.get_app_dir('piker') _watchlists_data_path = os.path.join(_config_dir, 'watchlists.json') +_data_mods = [ + 'piker.brokers.core', + 'piker.brokers.data', +] @click.command() @@ -33,7 +37,7 @@ def pikerd(loglevel, host, tl): """ get_console_log(loglevel) tractor.run_daemon( - rpc_module_paths=['piker.brokers.data'], + rpc_module_paths=_data_mods, name='brokerd', loglevel=loglevel if tl else None, ) @@ -133,7 +137,7 @@ async def maybe_spawn_brokerd_as_subactor(sleep=0.5, tries=10, loglevel=None): "No broker daemon could be found, spawning brokerd..") portal = await nursery.start_actor( 'brokerd', - rpc_module_paths=['piker.brokers.data'], + rpc_module_paths=_data_mods, loglevel=loglevel, ) yield portal @@ -144,7 +148,7 @@ async def maybe_spawn_brokerd_as_subactor(sleep=0.5, tries=10, loglevel=None): help='Broker backend to use') @click.option('--loglevel', '-l', default='warning', help='Logging level') @click.option('--tl', is_flag=True, help='Enable tractor logging') -@click.option('--rate', '-r', default=5, help='Quote rate limit') +@click.option('--rate', '-r', default=4, help='Quote rate limit') @click.option('--test', '-t', help='Test quote stream file') @click.option('--dhost', '-dh', default='127.0.0.1', help='Daemon host address to connect to') @@ -358,3 +362,36 @@ def optsquote(loglevel, broker, symbol, df_output, date): click.echo(df) else: click.echo(colorize_json(quotes)) + + +@cli.command() +@click.option('--broker', '-b', default=DEFAULT_BROKER, + help='Broker backend to use') +@click.option('--loglevel', '-l', default='warning', help='Logging level') +@click.option('--tl', is_flag=True, help='Enable tractor logging') +@click.option('--date', '-d', help='Contracts expiry date') +@click.option('--test', '-t', help='Test quote stream file') +@click.option('--rate', '-r', default=4, help='Logging level') +@click.argument('symbol', required=True) +def optschain(loglevel, broker, symbol, date, tl, rate, test): + """Start the real-time option chain UI. + """ + from .ui.option_chain import _async_main + log = get_console_log(loglevel) # activate console logging + brokermod = get_brokermod(broker) + + async def main(tries): + async with maybe_spawn_brokerd_as_subactor( + tries=tries, loglevel=loglevel + ) as portal: + # run app "main" + await _async_main( + symbol, portal, + brokermod, rate=rate, test=test, + ) + + tractor.run( + partial(main, tries=1), + name='kivy-options-chain', + loglevel=loglevel if tl else None, + ) diff --git a/piker/ui/option_chain.py b/piker/ui/option_chain.py new file mode 100644 index 00000000..5e1764cf --- /dev/null +++ b/piker/ui/option_chain.py @@ -0,0 +1,275 @@ +""" +options: a real-time option chain. + +Launch with ``piker options ``. +""" +import types +from functools import partial + +import trio +import tractor +from kivy.uix.boxlayout import BoxLayout +from kivy.lang import Builder +from kivy.app import async_runTouchApp +from kivy.core.window import Window + +from ..log import get_logger +from ..brokers.core import contracts +from .pager import PagerView + +from .monitor import Row, HeaderCell, Cell, TickerTable, update_quotes + + +log = get_logger('option_chain') + + +async def modify_symbol(symbol): + pass + + +class ExpiryButton(HeaderCell): + def on_press(self, value=None): + log.info(f"Clicked {self}") + + +class StrikeCell(Cell): + """Strike cell""" + + +_no_display = ['symbol', 'contract_type', 'strike', 'time', 'open'] + + +class StrikeRow(BoxLayout): + """A 'row' composed of two ``Row``s sandwiching a + ``StrikeCell`. + """ + def __init__(self, strike, **kwargs): + super().__init__(orientation='horizontal', **kwargs) + self.strike = strike + # store 2 rows: 1 for call, 1 for put + self._sub_rows = {} + self.table = None + + def append_sub_row( + self, + record: dict, + bidasks=None, + headers=(), + table=None, + **kwargs, + ) -> None: + if self.is_populated(): + raise TypeError(f"{self} can only append two sub-rows?") + + # the 'contract_type' determines whether this + # is a put or call row + contract_type = record['contract_type'] + + # reverse order of call side cells + if contract_type == 'call': + record = dict(list(reversed(list(record.items())))) + + row = Row( + record, + bidasks=bidasks, + headers=headers, + table=table, + no_cell=_no_display, + **kwargs + ) + # reassign widget for when rendered in the update loop + row.widget = self + self._sub_rows[contract_type] = row + if self.is_populated(): + # calls on the left + self.add_widget(self._sub_rows['call']) + # strikes in the middle + self.add_widget( + StrikeCell( + key=self.strike, + text=str(self.strike), + is_header=True, + # make centre strike cell nice and small + size_hint=(1/8., 1), + ) + ) + # puts on the right + self.add_widget(self._sub_rows['put']) + + def is_populated(self): + """Bool determing if both a put and call subrow have beed appended. + """ + return len(self._sub_rows) == 2 + + def update(self, record, displayable): + self._sub_rows[record['contract_type']].update( + record, displayable) + + +async def _async_main( + symbol: str, + portal: tractor._portal.Portal, + brokermod: types.ModuleType, + rate: int = 4, + test: bool = False +) -> None: + '''Launch kivy app + all other related tasks. + + This is started with cli cmd `piker options`. + ''' + # retreive all contracts + all_contracts = await contracts(brokermod, symbol) + first_expiry = next(iter(all_contracts)).expiry + + if test: + # stream from a local test file + quote_gen = await portal.run( + "piker.brokers.data", 'stream_from_file', + filename=test + ) + else: + # start live streaming from broker daemon + quote_gen = await portal.run( + "piker.brokers.data", + 'start_quote_stream', + broker=brokermod.name, + symbols=[(symbol, first_expiry)], + feed_type='option', + ) + + # get first quotes response + log.debug("Waiting on first quote...") + quotes = await quote_gen.__anext__() + records, displayables = zip(*[ + brokermod.format_option_quote(quote, {}) + for quote in quotes.values() + ]) + + # define bid-ask "stacked" cells + # (TODO: needs some rethinking and renaming for sure) + bidasks = brokermod._option_bidasks + + # build out UI + title = f"option chain: {symbol}\t(press ? for help)" + Window.set_title(title) + + # use `monitor` styling for now + from .monitor import _kv + Builder.load_string(_kv) + + # the master container + container = BoxLayout(orientation='vertical', spacing=0) + + # TODO: figure out how to compact these buttons + expiries = { + key.expiry: key.expiry[:key.expiry.find('T')] + for key in all_contracts + } + expiry_buttons = Row( + record=expiries, + headers=expiries, + is_header=True, + size_hint=(1, None), + cell_type=ExpiryButton, + ) + # top row of expiry buttons + container.add_widget(expiry_buttons) + + # figure out header fields for each table based on quote keys + headers = displayables[0].keys() + header_row = StrikeRow(strike='strike', size_hint=(1, None)) + header_record = {key: key for key in headers} + header_record['contract_type'] = 'put' + header_row.append_sub_row( + header_record, + headers=headers, + bidasks=bidasks, + is_header=True, + size_hint=(1, None), + + ) + header_record['contract_type'] = 'call' + header_row.append_sub_row( + header_record, + headers=headers, + bidasks=bidasks, + is_header=True, + size_hint=(1, None), + + ) + container.add_widget(header_row) + + table = TickerTable( + sort_key='strike', + cols=1, + size_hint=(1, None), + ) + header_row.table = table + + strike_rows = {} + for record, display in zip(sorted( + records, + key=lambda q: q['strike'], + ), displayables): + strike = record['strike'] + strike_row = strike_rows.setdefault( + strike, StrikeRow(strike)) + strike_row.append_sub_row( + record, + bidasks=bidasks, + table=table, + ) + if strike_row.is_populated(): + # We must fill out the the table's symbol2rows manually + # using each contracts "symbol" so that the quote updater + # task can look up the right row to update easily + # See update_quotes() and ``Row`` for details. + for contract_type, row in strike_row._sub_rows.items(): + table.symbols2rows[row._last_record['symbol']] = row + + table.append_row(symbol, strike_row) + + async with trio.open_nursery() as nursery: + # set up a pager view for large ticker lists + table.bind(minimum_height=table.setter('height')) + pager = PagerView( + container=container, + contained=table, + nursery=nursery + ) + container.add_widget(pager) + widgets = { + 'root': container, + 'container': container, + 'table': table, + 'expiry_buttons': expiry_buttons, + 'pager': pager, + } + nursery.start_soon( + partial( + update_quotes, + nursery, + brokermod.format_option_quote, + widgets, + quote_gen, + symbol_data={}, + first_quotes=quotes, + ) + ) + try: + # Trio-kivy entry point. + await async_runTouchApp(widgets['root']) # run kivy + finally: + await quote_gen.aclose() # cancel aysnc gen call + # un-subscribe from symbols stream (cancel if brokerd + # was already torn down - say by SIGINT) + with trio.move_on_after(0.2): + await portal.run( + "piker.brokers.data", 'modify_quote_stream', + broker=brokermod.name, + feed_type='option', + symbols=[] + ) + + # cancel GUI update task + nursery.cancel_scope.cancel() From 01c0551a7f154fc6692d9bedeb8d3249feecae42 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 11 Dec 2018 15:20:24 -0500 Subject: [PATCH 07/42] Don't display greeks besides delta for now --- piker/brokers/questrade.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index c9f2cc9e..50ee4ce6 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -25,7 +25,10 @@ log = get_logger(__name__) _refresh_token_ep = 'https://login.questrade.com/oauth2/' _version = 'v1' -_rate_limit = 4 # queries/sec + +# stock queries/sec +# it seems 4 rps is best we can do total +_rate_limit = 4 class QuestradeError(Exception): @@ -414,8 +417,10 @@ async def get_client() -> Client: try: log.debug("Check time to ensure access token is valid") try: - await client.api.time() - except Exception: + # await client.api.time() + quote = await client.quote(['RY.TO']) + except Exception as err: + # import pdb; pdb.set_trace() # access token is likely no good log.warn(f"Access token {client.access_data['access_token']} seems" f" expired, forcing refresh") @@ -642,10 +647,10 @@ _qt_option_keys = { # "expiry": "expiry", # "delay": 0, "delta": ('delta', partial(round, ndigits=3)), - "gamma": ('gama', partial(round, ndigits=3)), - "rho": ('rho', partial(round, ndigits=3)), - "theta": ('theta', partial(round, ndigits=3)), - "vega": ('vega', partial(round, ndigits=3)), + # "gamma": ('gama', partial(round, ndigits=3)), + # "rho": ('rho', partial(round, ndigits=3)), + # "theta": ('theta', partial(round, ndigits=3)), + # "vega": ('vega', partial(round, ndigits=3)), '$ vol': ('$ vol', humanize), 'volume': ('vol', humanize), # "2021-01-15T00:00:00.000000-05:00", @@ -666,7 +671,10 @@ _qt_option_keys = { # "underlyingId": 8297492, "symbol": 'symbol', "contract_type": 'contract_type', - "volatility": ('volatility', partial(round, ndigits=3)), + "volatility": ( + 'volatility', + lambda v: '{}%'.format(round(v, ndigits=2)) + ), "strike": 'strike', } From 66ecb4c0cbd209cbb840b4f49edf68424cd0d397 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 11 Dec 2018 15:21:12 -0500 Subject: [PATCH 08/42] Use a `trio.Event` to guarantee respawning of data feed task --- piker/brokers/data.py | 98 +++++++++++++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 31 deletions(-) diff --git a/piker/brokers/data.py b/piker/brokers/data.py index b6e5ab6a..b9431654 100644 --- a/piker/brokers/data.py +++ b/piker/brokers/data.py @@ -108,6 +108,9 @@ async def stream_quotes( await trio.sleep(delay) +# TODO: at this point probably just just make this a class and +# a lot of these functions should be methods. It will definitely +# make stateful UI apps easier to implement class DataFeed(typing.NamedTuple): """A per broker "data feed" container. @@ -118,7 +121,7 @@ class DataFeed(typing.NamedTuple): client: object exit_stack: contextlib.AsyncExitStack quoter_keys: List[str] = ['stock', 'option'] - tasks: Dict[str, trio._core._run.Task] = dict.fromkeys( + tasks: Dict[str, trio.Event] = dict.fromkeys( quoter_keys, False) quoters: Dict[str, typing.Coroutine] = {} subscriptions: Dict[str, Dict[str, set]] = {'option': {}, 'stock': {}} @@ -234,13 +237,19 @@ async def smoke_quote(get_quotes, tickers, broker): ########################################### -async def modify_quote_stream(broker, feed_type, symbols, chan=None, cid=None): +def modify_quote_stream(broker, feed_type, symbols, chan, cid): """Absolute symbol subscription list for each quote stream. Effectively a symbol subscription api. """ log.info(f"{chan} changed symbol subscription to {symbols}") - feed = await get_cached_feed(broker) + ss = tractor.current_actor().statespace + feed = ss['feeds'].get(broker) + if feed is None: + raise RuntimeError( + "`get_cached_feed()` must be called before modifying its stream" + ) + symbols2chans = feed.subscriptions[feed_type] # update map from each symbol to requesting client's chan for ticker in symbols: @@ -299,6 +308,7 @@ async def start_quote_stream( diff_cached: bool = True, chan: tractor.Channel = None, cid: str = None, + rate: int = 3, ) -> None: """Handle per-broker quote stream subscriptions using a "lazy" pub-sub pattern. @@ -311,7 +321,6 @@ async def start_quote_stream( # set log level after fork get_console_log(actor.loglevel) # pull global vars from local actor - ss = actor.statespace symbols = list(symbols) log.info( f"{chan.uid} subscribed to {broker} for symbols {symbols}") @@ -337,38 +346,65 @@ async def start_quote_stream( 'option', await feed.mod.option_quoter(feed.client, symbols) ) - - # update map from each symbol to requesting client's chan - await modify_quote_stream(broker, feed_type, symbols, chan, cid) - try: - if not feed.tasks.get(feed_type): - # no data feeder task yet; so start one - respawn = True + # update map from each symbol to requesting client's chan + modify_quote_stream(broker, feed_type, symbols, chan, cid) + + # event indicating that task was started and then killed + task_is_dead = feed.tasks.get(feed_type) + if task_is_dead is False: + task_is_dead = trio.Event() + task_is_dead.set() + feed.tasks[feed_type] = task_is_dead + + if not task_is_dead.is_set(): + # block and let existing feed task deliver + # stream data until it is cancelled in which case + # we'll take over and spawn it again + await task_is_dead.wait() + # client channel was likely disconnected + # but we still want to keep the broker task + # alive if there are other consumers (including + # ourselves) + if any(symbols2chans.values()): + log.warn( + f"Data feed task for {feed.mod.name} was cancelled but" + f" there are still active clients, respawning") + + # no data feeder task yet; so start one + respawn = True + while respawn: + respawn = False log.info(f"Spawning data feed task for {feed.mod.name}") - while respawn: - respawn = False - try: - async with trio.open_nursery() as nursery: - nursery.start_soon( - partial( - fan_out_to_chans, feed, get_quotes, - symbols2chans, - diff_cached=diff_cached, - cid=cid - ) + try: + async with trio.open_nursery() as nursery: + nursery.start_soon( + partial( + fan_out_to_chans, feed, get_quotes, + symbols2chans, + diff_cached=diff_cached, + cid=cid, + rate=rate, ) - feed.tasks[feed_type] = True - except trio.BrokenResourceError: - log.exception("Respawning failed data feed task") - respawn = True - # unblocks when no more symbols subscriptions exist and the - # quote streamer task terminates (usually because another call - # was made to `modify_quoter` to unsubscribe from streaming - # symbols) + ) + # it's alive! + task_is_dead.clear() + + except trio.BrokenResourceError: + log.exception("Respawning failed data feed task") + respawn = True + + # unblocks when no more symbols subscriptions exist and the + # quote streamer task terminates (usually because another call + # was made to `modify_quoter` to unsubscribe from streaming + # symbols) finally: log.info(f"Terminated {feed_type} quoter task for {feed.mod.name}") - feed.tasks.pop(feed_type) + task_is_dead.set() + + # if we're cancelled externally unsubscribe our quote feed + modify_quote_stream(broker, feed_type, [], chan, cid) + # if there are truly no more subscriptions with this broker # drop from broker subs dict if not any(symbols2chans.values()): From e7378538f69e1e78f05940046fbdf8f7626c2a82 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 11 Dec 2018 15:21:45 -0500 Subject: [PATCH 09/42] Limit option chain to 1 rps --- piker/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/piker/cli.py b/piker/cli.py index e282f669..8e4db81e 100644 --- a/piker/cli.py +++ b/piker/cli.py @@ -148,7 +148,7 @@ async def maybe_spawn_brokerd_as_subactor(sleep=0.5, tries=10, loglevel=None): help='Broker backend to use') @click.option('--loglevel', '-l', default='warning', help='Logging level') @click.option('--tl', is_flag=True, help='Enable tractor logging') -@click.option('--rate', '-r', default=4, help='Quote rate limit') +@click.option('--rate', '-r', default=3, help='Quote rate limit') @click.option('--test', '-t', help='Test quote stream file') @click.option('--dhost', '-dh', default='127.0.0.1', help='Daemon host address to connect to') @@ -371,7 +371,7 @@ def optsquote(loglevel, broker, symbol, df_output, date): @click.option('--tl', is_flag=True, help='Enable tractor logging') @click.option('--date', '-d', help='Contracts expiry date') @click.option('--test', '-t', help='Test quote stream file') -@click.option('--rate', '-r', default=4, help='Logging level') +@click.option('--rate', '-r', default=1, help='Logging level') @click.argument('symbol', required=True) def optschain(loglevel, broker, symbol, date, tl, rate, test): """Start the real-time option chain UI. @@ -387,7 +387,9 @@ def optschain(loglevel, broker, symbol, date, tl, rate, test): # run app "main" await _async_main( symbol, portal, - brokermod, rate=rate, test=test, + brokermod, + rate=rate, + test=test, ) tractor.run( From e1be80e9e0da48933f95a3534ab77aaf27f56aef Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 11 Dec 2018 15:22:34 -0500 Subject: [PATCH 10/42] Subscription teardown is done server side on disconnect now --- piker/ui/monitor.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/piker/ui/monitor.py b/piker/ui/monitor.py index 9d15b257..de371c06 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/monitor.py @@ -424,7 +424,7 @@ class TickerTable(GridLayout): symbol name. Most naive algo possible for the moment. """ for symbol, row in self.symbols2rows.items(): - if patt in symbol: + if patt in symbol: yield symbol, row def get_row(self, symbol: str) -> Row: @@ -546,7 +546,7 @@ async def _async_main( portal: tractor._portal.Portal, tickers: List[str], brokermod: ModuleType, - rate: int, + rate: int = 3, test: bool = False ) -> None: '''Launch kivy app + all other related tasks. @@ -565,8 +565,12 @@ async def _async_main( else: # start live streaming from broker daemon quote_gen = await portal.run( - "piker.brokers.data", 'start_quote_stream', - broker=brokermod.name, symbols=tickers) + "piker.brokers.data", + 'start_quote_stream', + broker=brokermod.name, + symbols=tickers, + rate=3, + ) # subscribe for tickers (this performs a possible filtering # where invalid symbols are discarded) @@ -657,16 +661,7 @@ async def _async_main( # Trio-kivy entry point. await async_runTouchApp(widgets['root']) # run kivy finally: - await quote_gen.aclose() # cancel aysnc gen call - # un-subscribe from symbols stream (cancel if brokerd - # was already torn down - say by SIGINT) - with trio.move_on_after(0.2): - await portal.run( - "piker.brokers.data", 'modify_quote_stream', - broker=brokermod.name, - feed_type='stock', - symbols=[] - ) - + # cancel aysnc gen call + await quote_gen.aclose() # cancel GUI update task nursery.cancel_scope.cancel() From 7b5c73bb45b36a94500afb3a93c4656c97e1dc35 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 11 Dec 2018 17:09:36 -0500 Subject: [PATCH 11/42] Use pythonic sequence splitting with `zip()` --- piker/brokers/questrade.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 50ee4ce6..0c69c932 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -504,8 +504,7 @@ async def option_quoter(client: Client, tickers: List[str]): ``(symbol_date_1, symbol_date_2, ... , symbol_date_n)`` return a contract dict. """ - symbols = map(itemgetter(0), sym_date_pairs) - dates = map(itemgetter(1), sym_date_pairs) + symbols, dates = zip(*sym_date_pairs) contracts = await client.get_all_contracts(symbols) selected = {} for key, val in contracts.items(): From 743ca6bfe3d12ea937001c4947a55a47fcb7493c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 11 Dec 2018 17:09:59 -0500 Subject: [PATCH 12/42] Log quotes even without caching --- piker/brokers/data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/brokers/data.py b/piker/brokers/data.py index b9431654..af5c10b3 100644 --- a/piker/brokers/data.py +++ b/piker/brokers/data.py @@ -90,6 +90,7 @@ async def stream_quotes( new_quotes.append(quote) else: new_quotes = quotes + log.info(f"Delivering quotes:\n{quotes}") yield new_quotes From 1d1be9dd77f4b5aaa3fd58a10817f6d2d119469a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 11 Dec 2018 17:10:36 -0500 Subject: [PATCH 13/42] Include option stream subscription change in test --- tests/test_questrade.py | 49 +++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/tests/test_questrade.py b/tests/test_questrade.py index 4c3ac059..39b64cc7 100644 --- a/tests/test_questrade.py +++ b/tests/test_questrade.py @@ -51,6 +51,7 @@ _ex_quotes = { 'askSize': 0, 'bidPrice': None, 'bidSize': 0, + 'contract_type': 'call', 'delay': 0, 'delta': -0.212857, "expiry": "2021-01-15T00:00:00.000000-05:00", @@ -200,6 +201,7 @@ async def stream_option_chain(portal, symbols): broker='questrade', symbols=[sub], feed_type='option', + rate=4, diff_cached=False, ) try: @@ -223,15 +225,34 @@ async def stream_option_chain(portal, symbols): count += 1 if count == loops: break + + # switch the subscription and make sure + # stream is still working + sub = subs_keys[1] + await agen.aclose() + agen = await portal.run( + 'piker.brokers.data', + 'start_quote_stream', + broker='questrade', + symbols=[sub], + feed_type='option', + rate=4, + diff_cached=False, + ) + + await agen.__anext__() + with trio.fail_after(2.1): + loops = 8 + count = 0 + async for quotes in agen: + for symbol, quote in quotes.items(): + assert quote['key'] == sub + count += 1 + if count == loops: + break finally: # unsub - await portal.run( - 'piker.brokers.data', - 'modify_quote_stream', - broker='questrade', - feed_type='option', - symbols=[], - ) + await agen.aclose() async def stream_stocks(portal, symbols): @@ -252,13 +273,7 @@ async def stream_stocks(portal, symbols): break finally: # unsub - await portal.run( - 'piker.brokers.data', - 'modify_quote_stream', - broker='questrade', - feed_type='stock', - symbols=[], - ) + await agen.aclose() @pytest.mark.parametrize( @@ -286,9 +301,9 @@ async def test_quote_streaming(tmx_symbols, loglevel, stream_what): 'piker.brokers.core' ], ) - async with trio.open_nursery() as n: - for func in stream_what: - n.start_soon(func, portal, tmx_symbols) + async with trio.open_nursery() as n: + for func in stream_what: + n.start_soon(func, portal, tmx_symbols) # stop all spawned subactors await nursery.cancel() From 9e4786e62fc077d1d33b07551a7cd711beb7c8ff Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 13 Dec 2018 13:04:05 -0500 Subject: [PATCH 14/42] Initial dynamic option chain UI draft There's still a ton to polish (and some bugs to fix) but this is a first working draft of a real-time option chain! Insights and todos: - `kivy` widgets need to be cached and reused (eg. rows, cells, etc.) for speed since it seems creating new ones constantly is quite taxing on the CPU - the chain will tear down and re-setup the option data feed stream each time a different contract expiry button set is clicked - there's still some weird bug with row highlighting where it seems rows added from a new expiry set (which weren't previously rendered) aren't being highlighted reliably --- piker/ui/monitor.py | 22 +- piker/ui/option_chain.py | 453 +++++++++++++++++++++++++++------------ 2 files changed, 333 insertions(+), 142 deletions(-) diff --git a/piker/ui/monitor.py b/piker/ui/monitor.py index de371c06..180c88c6 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/monitor.py @@ -304,6 +304,9 @@ class Row(GridLayout, HoverBehavior): cell.key = key self._cell_widgets[key] = cell + def iter_cells(self): + return self._cell_widgets.items() + def get_cell(self, key): return self._cell_widgets.get(key) @@ -356,7 +359,7 @@ class Row(GridLayout, HoverBehavior): """Highlight layout on enter. """ log.debug( - f"Entered row {type(self)} through {self.border_point}") + f"Entered row {self} through {self.border_point}") # don't highlight header row if getattr(self, 'is_header', None): self.hovered = False @@ -365,7 +368,7 @@ class Row(GridLayout, HoverBehavior): """Un-highlight layout on exit. """ log.debug( - f"Left row {type(self)} through {self.border_point}") + f"Left row {self} through {self.border_point}") class TickerTable(GridLayout): @@ -424,7 +427,7 @@ class TickerTable(GridLayout): symbol name. Most naive algo possible for the moment. """ for symbol, row in self.symbols2rows.items(): - if patt in symbol: + if patt in symbol: yield symbol, row def get_row(self, symbol: str) -> Row: @@ -442,10 +445,12 @@ async def update_quotes( widgets: dict, agen: AsyncGeneratorType, symbol_data: dict, - first_quotes: dict + first_quotes: dict, + task_status: trio._core._run._TaskStatus = trio.TASK_STATUS_IGNORED, ): """Process live quotes by updating ticker rows. """ + log.debug("Initializing UI update loop") table = widgets['table'] flash_keys = {'low', 'high'} @@ -521,9 +526,8 @@ async def update_quotes( color_row(row, record, {}) cache[sym] = row - # render all rows once up front - table.render_rows(cache) - + log.debug("Finished initializing update loop") + task_status.started() # real-time cell update loop async for quotes in agen: # new quotes data only for symbol, quote in quotes.items(): @@ -538,6 +542,7 @@ async def update_quotes( log.debug("Waiting on quotes") log.warn("Data feed connection dropped") + # XXX: if we're cancelled this should never get called nursery.cancel_scope.cancel() @@ -617,7 +622,8 @@ async def _async_main( for ticker_record in first_quotes: table.append_row( ticker_record['symbol'], - Row(ticker_record, headers=('symbol',), bidasks=bidasks, table=table) + Row(ticker_record, headers=('symbol',), + bidasks=bidasks, table=table) ) # associate the col headers row with the ticker table even though diff --git a/piker/ui/option_chain.py b/piker/ui/option_chain.py index 5e1764cf..9a8b4d74 100644 --- a/piker/ui/option_chain.py +++ b/piker/ui/option_chain.py @@ -5,8 +5,11 @@ Launch with ``piker options ``. """ import types from functools import partial +from typing import Dict, List +# import typing import trio +from async_generator import asynccontextmanager import tractor from kivy.uix.boxlayout import BoxLayout from kivy.lang import Builder @@ -27,129 +30,328 @@ async def modify_symbol(symbol): pass -class ExpiryButton(HeaderCell): - def on_press(self, value=None): - log.info(f"Clicked {self}") - - class StrikeCell(Cell): """Strike cell""" _no_display = ['symbol', 'contract_type', 'strike', 'time', 'open'] +_strike_row_cache = {} +_strike_cell_cache = {} class StrikeRow(BoxLayout): """A 'row' composed of two ``Row``s sandwiching a ``StrikeCell`. """ + _row_cache = {} + def __init__(self, strike, **kwargs): super().__init__(orientation='horizontal', **kwargs) self.strike = strike # store 2 rows: 1 for call, 1 for put self._sub_rows = {} - self.table = None + self._widgets_added = False def append_sub_row( self, record: dict, + displayable: dict, bidasks=None, headers=(), table=None, **kwargs, ) -> None: - if self.is_populated(): - raise TypeError(f"{self} can only append two sub-rows?") + # if self.is_populated(): + # raise TypeError(f"{self} can only append two sub-rows?") # the 'contract_type' determines whether this # is a put or call row contract_type = record['contract_type'] - # reverse order of call side cells - if contract_type == 'call': - record = dict(list(reversed(list(record.items())))) + # We want to only create a few ``Row`` widgets as possible to + # speed up rendering; we cache sub rows after creation. + row = self._row_cache.get((self.strike, contract_type)) + if not row: + # reverse order of call side cells + if contract_type == 'call': + record = dict(list(reversed(list(record.items())))) + row = Row( + record, + bidasks=bidasks, + headers=headers, + table=table, + no_cell=_no_display, + **kwargs + ) + self._row_cache[(self.strike, contract_type)] = row + else: + # must update the internal cells + row.update(record, displayable) - row = Row( - record, - bidasks=bidasks, - headers=headers, - table=table, - no_cell=_no_display, - **kwargs - ) # reassign widget for when rendered in the update loop row.widget = self self._sub_rows[contract_type] = row - if self.is_populated(): + + if self.is_populated() and not self._widgets_added: # calls on the left self.add_widget(self._sub_rows['call']) - # strikes in the middle - self.add_widget( - StrikeCell( + strike_cell = _strike_cell_cache.setdefault( + self.strike, StrikeCell( key=self.strike, text=str(self.strike), is_header=True, # make centre strike cell nice and small - size_hint=(1/8., 1), + size_hint=(1/10., 1), ) ) + # strikes in the middle + self.add_widget(strike_cell) # puts on the right self.add_widget(self._sub_rows['put']) + self._widgets_added = True def is_populated(self): """Bool determing if both a put and call subrow have beed appended. """ return len(self._sub_rows) == 2 + def has_widgets(self): + return self._widgets_added + def update(self, record, displayable): self._sub_rows[record['contract_type']].update( record, displayable) -async def _async_main( - symbol: str, +class ExpiryButton(HeaderCell): + def on_press(self, value=None): + log.info(f"Clicked {self}") + if self.chain.sub[1] == self.key: + log.info(f"Clicked {self} is already selected") + return + log.info(f"Subscribing for {self.chain.sub}") + self.chain.start_displaying(self.chain.sub[0], self.key) + + +class DataFeed(object): + """Data feed client for streaming symbol data from a remote + broker data source. + """ + def __init__(self, portal, brokermod): + self.portal = portal + self.brokermod = brokermod + self.sub = None + self.quote_gen = None + + async def open_stream(self, symbols, rate=3, test=None): + if self.quote_gen is not None and symbols != self.sub: + log.info(f"Stopping existing subscription for {self.sub}") + await self.quote_gen.aclose() + self.sub = symbols + + if test: + # stream from a local test file + quote_gen = await self.portal.run( + "piker.brokers.data", 'stream_from_file', + filename=test + ) + else: + # start live streaming from broker daemon + quote_gen = await self.portal.run( + "piker.brokers.data", + 'start_quote_stream', + broker=self.brokermod.name, + symbols=symbols, + feed_type='option', + rate=rate, + ) + + # get first quotes response + log.debug(f"Waiting on first quote for {symbols}...") + quotes = await quote_gen.__anext__() + + self.quote_gen = quote_gen + self.first_quotes = quotes + # self.records = records + # self.displayables = displayables + return quote_gen, quotes + + +class OptionChain(object): + """A real-time options chain UI. + """ + def __init__( + self, + symbol: str, + expiry: str, + widgets: dict, + bidasks: Dict[str, List[str]], + feed: DataFeed, + rate: int = 1, + ): + self.sub = (symbol, expiry) + self.widgets = widgets + self.bidasks = bidasks + self._strikes2rows = {} + self._nursery = None + self.feed = feed + self._update_cs = None + # TODO: this should be moved down to the data feed layer + # right now it's only needed for the UI uupdate loop to cancel itself + self._first_quotes = None + + @asynccontextmanager + async def open_scope(self): + """Open an internal resource and update task scope required + to allow for dynamic real-time operation. + """ + # assign us to each expiry button + for key, button in ( + self.widgets['expiry_buttons']._cell_widgets.items() + ): + button.chain = self + + async with trio.open_nursery() as n: + self._nursery = n + n.start_soon(self.start_updating) + yield self + + self._nursery = None + await self.feed.quote_gen.aclose() + + def clear(self): + """Clear the strike rows from the internal table. + """ + table = self.widgets['table'] + table.clear_widgets() + for strike in self._strikes2rows.copy(): + self._strikes2rows.pop(strike) + + def render_rows(self, records, displayables): + """Render all strike rows in the internal table. + """ + log.debug("Rendering rows") + table = self.widgets['table'] + for record, display in zip( + sorted(records, key=lambda q: q['strike']), + displayables + ): + strike = record['strike'] + strike_row = _strike_row_cache.setdefault( + strike, StrikeRow(strike)) + strike_row.append_sub_row( + record, + display, + bidasks=self.bidasks, + table=table, + ) + if strike_row.is_populated(): + # We must fill out the the table's symbol2rows manually + # using each contracts "symbol" so that the quote updater + # task can look up the right row to update easily + # See update_quotes() and ``Row`` internals for details. + for contract_type, row in strike_row._sub_rows.items(): + symbol = row._last_record['symbol'] + table.symbols2rows[symbol] = row + + if strike not in self._strikes2rows: + # readding widgets is an error + table.add_widget(strike_row) + self._strikes2rows[strike] = strike_row + + log.debug("Finished rendering rows!") + + async def start_feed( + self, + symbol: str, + expiry: str, + # max QT rate per API customer is approx 4 rps + # and usually 3 rps is allocated to the stock monitor + rate: int = 1, + test: str = None + ): + if self.feed.sub != self.sub: + return await self.feed.open_stream([(symbol, expiry)], rate=rate) + else: + feed = self.feed + return feed.quote_gen, feed.first_quotes + + async def start_updating(self): + if self._update_cs: + self._update_cs.cancel() + await trio.sleep(0) + + self.clear() + + if self._nursery is None: + raise RuntimeError( + "You must call await `start()` first!") + + n = self._nursery + log.debug(f"Waiting on first_quotes for {self.sub}") + quote_gen, first_quotes = await self.start_feed(*self.sub) + log.debug(f"Got first_quotes for {self.sub}") + + # redraw the UI + records, displayables = zip(*[ + self.feed.brokermod.format_option_quote(quote, {}) + for quote in first_quotes.values() + ]) + self.render_rows(records, displayables) + + with trio.open_cancel_scope() as cs: + self._update_cs = cs + # start quote update loop + await n.start( + partial( + update_quotes, + self._nursery, + self.feed.brokermod.format_option_quote, + self.widgets, + quote_gen, + symbol_data={}, + first_quotes=first_quotes, + ) + ) + + def start_displaying(self, symbol, expiry): + self.sub = (symbol, expiry) + self._nursery.start_soon(self.start_updating) + + +async def new_chain_ui( portal: tractor._portal.Portal, + symbol: str, + expiry: str, + contracts, brokermod: types.ModuleType, - rate: int = 4, - test: bool = False + nursery: trio._core._run.Nursery, + rate: int = 1, ) -> None: - '''Launch kivy app + all other related tasks. - - This is started with cli cmd `piker options`. - ''' - # retreive all contracts - all_contracts = await contracts(brokermod, symbol) - first_expiry = next(iter(all_contracts)).expiry - - if test: - # stream from a local test file - quote_gen = await portal.run( - "piker.brokers.data", 'stream_from_file', - filename=test - ) - else: - # start live streaming from broker daemon - quote_gen = await portal.run( - "piker.brokers.data", - 'start_quote_stream', - broker=brokermod.name, - symbols=[(symbol, first_expiry)], - feed_type='option', - ) - - # get first quotes response - log.debug("Waiting on first quote...") - quotes = await quote_gen.__anext__() - records, displayables = zip(*[ - brokermod.format_option_quote(quote, {}) - for quote in quotes.values() - ]) - + """Create and return a new option chain UI. + """ + widgets = {} # define bid-ask "stacked" cells # (TODO: needs some rethinking and renaming for sure) bidasks = brokermod._option_bidasks - # build out UI + feed = DataFeed(portal, brokermod) + chain = OptionChain( + symbol, + expiry, + widgets, + bidasks, + feed, + rate=rate, + ) + + quote_gen, first_quotes = await chain.start_feed(symbol, expiry) + records, displayables = zip(*[ + brokermod.format_option_quote(quote, {}) + for quote in first_quotes.values() + ]) + + # build out root UI title = f"option chain: {symbol}\t(press ? for help)" Window.set_title(title) @@ -163,7 +365,7 @@ async def _async_main( # TODO: figure out how to compact these buttons expiries = { key.expiry: key.expiry[:key.expiry.find('T')] - for key in all_contracts + for key in contracts } expiry_buttons = Row( record=expiries, @@ -182,94 +384,77 @@ async def _async_main( header_record['contract_type'] = 'put' header_row.append_sub_row( header_record, - headers=headers, - bidasks=bidasks, - is_header=True, - size_hint=(1, None), - - ) - header_record['contract_type'] = 'call' - header_row.append_sub_row( header_record, headers=headers, bidasks=bidasks, is_header=True, size_hint=(1, None), - + ) + header_record['contract_type'] = 'call' + header_row.append_sub_row( + header_record, + header_record, + headers=headers, + bidasks=bidasks, + is_header=True, + size_hint=(1, None), ) container.add_widget(header_row) - table = TickerTable( sort_key='strike', cols=1, size_hint=(1, None), ) header_row.table = table + table.bind(minimum_height=table.setter('height')) + pager = PagerView( + container=container, + contained=table, + nursery=nursery + ) + container.add_widget(pager) + widgets.update({ + 'root': container, + 'container': container, + 'table': table, + 'expiry_buttons': expiry_buttons, + 'pager': pager, + }) + return chain - strike_rows = {} - for record, display in zip(sorted( - records, - key=lambda q: q['strike'], - ), displayables): - strike = record['strike'] - strike_row = strike_rows.setdefault( - strike, StrikeRow(strike)) - strike_row.append_sub_row( - record, - bidasks=bidasks, - table=table, - ) - if strike_row.is_populated(): - # We must fill out the the table's symbol2rows manually - # using each contracts "symbol" so that the quote updater - # task can look up the right row to update easily - # See update_quotes() and ``Row`` for details. - for contract_type, row in strike_row._sub_rows.items(): - table.symbols2rows[row._last_record['symbol']] = row - table.append_row(symbol, strike_row) +async def _async_main( + symbol: str, + portal: tractor._portal.Portal, + brokermod: types.ModuleType, + rate: int = 1, + test: bool = False +) -> None: + '''Launch kivy app + all other related tasks. + + This is started with cli cmd `piker options`. + ''' + # retreive all contracts just because we need a default when the + # UI starts up + all_contracts = await contracts(brokermod, symbol) + # start streaming soonest contract by default + first_expiry = next(iter(all_contracts)).expiry async with trio.open_nursery() as nursery: # set up a pager view for large ticker lists - table.bind(minimum_height=table.setter('height')) - pager = PagerView( - container=container, - contained=table, - nursery=nursery + chain = await new_chain_ui( + portal, + symbol, + first_expiry, + all_contracts, + brokermod, + nursery, + rate=rate, ) - container.add_widget(pager) - widgets = { - 'root': container, - 'container': container, - 'table': table, - 'expiry_buttons': expiry_buttons, - 'pager': pager, - } - nursery.start_soon( - partial( - update_quotes, - nursery, - brokermod.format_option_quote, - widgets, - quote_gen, - symbol_data={}, - first_quotes=quotes, - ) - ) - try: - # Trio-kivy entry point. - await async_runTouchApp(widgets['root']) # run kivy - finally: - await quote_gen.aclose() # cancel aysnc gen call - # un-subscribe from symbols stream (cancel if brokerd - # was already torn down - say by SIGINT) - with trio.move_on_after(0.2): - await portal.run( - "piker.brokers.data", 'modify_quote_stream', - broker=brokermod.name, - feed_type='option', - symbols=[] - ) - - # cancel GUI update task - nursery.cancel_scope.cancel() + async with chain.open_scope(): + try: + # Trio-kivy entry point. + await async_runTouchApp(chain.widgets['root']) # run kivy + finally: + # cancel GUI update task + nursery.cancel_scope.cancel() From 948ee3cadf124ec85f4a1a6d8517bb62187f3878 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 13 Dec 2018 13:11:07 -0500 Subject: [PATCH 15/42] Cache contracts lookup once at startup --- piker/brokers/questrade.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 0c69c932..2ec2687e 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -499,15 +499,20 @@ async def option_quoter(client: Client, tickers: List[str]): raise ValueError(f'Option subscription format is (symbol, expiry)') @async_lifo_cache(maxsize=128) - async def get_contract_by_date(sym_date_pairs: Tuple[Tuple[str, str]]): + async def get_contract_by_date( + sym_date_pairs: Tuple[Tuple[str, str]], + _contract_cache: dict = {} + ): """For each tuple, ``(symbol_date_1, symbol_date_2, ... , symbol_date_n)`` return a contract dict. """ symbols, dates = zip(*sym_date_pairs) - contracts = await client.get_all_contracts(symbols) + if not _contract_cache: + contracts = await client.get_all_contracts(symbols) + _contract_cache.update(contracts) selected = {} - for key, val in contracts.items(): + for key, val in _contract_cache.items(): if key.expiry in dates: selected[key] = val From e3a3a8765cc037eb0c4d2e9c451585f37d6d306b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 15 Dec 2018 16:26:21 -0500 Subject: [PATCH 16/42] Remove destroyed widgets from mouse over list --- piker/ui/kivy/mouse_over.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/piker/ui/kivy/mouse_over.py b/piker/ui/kivy/mouse_over.py index 80a67d63..8b472ce7 100644 --- a/piker/ui/kivy/mouse_over.py +++ b/piker/ui/kivy/mouse_over.py @@ -89,14 +89,17 @@ class MouseOverBehavior(object): def __init__(self, **kwargs): self.register_event_type('on_enter') self.register_event_type('on_leave') - MouseOverBehavior._widgets.append(self) super().__init__(**kwargs) Window.bind(mouse_pos=self._on_mouse_pos) + self._widgets.append(self) + + def __del__(self): + MouseOverBehavior.remove(self) @classmethod # try throttling to 1ms latency (doesn't seem to work - # best I can get is 0.01...) - @triggered(timeout=0.001, interval=False) + # best I can get is 0.01 ?) + @triggered(timeout=0.015, interval=False) def _on_mouse_pos(cls, *args): log.debug(f"{cls} time since last call: {time.time() - cls._last_time}") cls._last_time = time.time() @@ -107,10 +110,11 @@ class MouseOverBehavior(object): pos = args[1] # Next line to_widget allow to compensate for relative layout - for widget in cls._widgets.copy(): + for widget in cls._widgets: w_coords = widget.to_widget(*pos) inside = widget.collide_point(*w_coords) if inside and widget.hovered: + log.debug('already hovered') return elif inside: # un-highlight the last highlighted From 70435e3b15628696d201a14cd59bcae112691961 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 15 Dec 2018 16:26:54 -0500 Subject: [PATCH 17/42] Always push an option smoke quote for UI init --- piker/brokers/data.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/piker/brokers/data.py b/piker/brokers/data.py index af5c10b3..e449a770 100644 --- a/piker/brokers/data.py +++ b/piker/brokers/data.py @@ -146,22 +146,29 @@ async def fan_out_to_chans( async def request(): """Get quotes for current symbol subscription set. """ - return await get_quotes(list(symbols2chans.keys())) + symbols = list(symbols2chans.keys()) + if symbols: + # subscription can be changed at any time + return await get_quotes(symbols) + else: + return () async for quotes in stream_quotes( feed.mod, request, rate, diff_cached=diff_cached, ): chan_payloads = {} + payload = {} for quote in quotes: # is this too QT specific? symbol = quote['symbol'] + payload[symbol] = quote # set symbol quotes for each subscriber for chan, cid in symbols2chans.get(quote['key'], set()): chan_payloads.setdefault( chan, - {'yield': {}, 'cid': cid} - )['yield'][symbol] = quote + {'yield': payload, 'cid': cid} + ) # deliver to each subscriber (fan out) if chan_payloads: @@ -347,6 +354,12 @@ async def start_quote_stream( 'option', await feed.mod.option_quoter(feed.client, symbols) ) + payload = { + quote['symbol']: quote + for quote in await get_quotes(symbols) + } + # push initial smoke quote response for client initialization + await chan.send({'yield': payload, 'cid': cid}) try: # update map from each symbol to requesting client's chan modify_quote_stream(broker, feed_type, symbols, chan, cid) From 721e3803b274b6b7ec8d2ce5b896a0d674e4d7de Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 15 Dec 2018 16:27:41 -0500 Subject: [PATCH 18/42] Shorter title of IV: implied volatility --- piker/brokers/questrade.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 2ec2687e..7fa00279 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -676,8 +676,8 @@ _qt_option_keys = { "symbol": 'symbol', "contract_type": 'contract_type', "volatility": ( - 'volatility', - lambda v: '{}%'.format(round(v, ndigits=2)) + 'IV %', + lambda v: '{}'.format(round(v, ndigits=2)) ), "strike": 'strike', } From 07eb8ae5e09caf1d0a6d4832287c18a23e445762 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 15 Dec 2018 16:28:28 -0500 Subject: [PATCH 19/42] Use binary search (bisection) to sort table rows This is an optimization to improve performance when the UI is fed real time data. Instead of resorting all rows on every quote update, only re-render when the sort key appears in the quote data, and further, only resort rows which are changed using bisection based widget insertion to avoid having `kivy` re-add widgets (and thus re-render graphics) more often than absolutely necessary. --- piker/ui/monitor.py | 152 ++++++++++++++++++++++++++------------------ 1 file changed, 90 insertions(+), 62 deletions(-) diff --git a/piker/ui/monitor.py b/piker/ui/monitor.py index 180c88c6..be90f31b 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/monitor.py @@ -8,6 +8,7 @@ Launch with ``piker monitor ``. from itertools import chain from types import ModuleType, AsyncGeneratorType from typing import List, Callable, Dict +from bisect import bisect import trio import tractor @@ -19,6 +20,7 @@ from kivy.lang import Builder from kivy import utils from kivy.app import async_runTouchApp from kivy.core.window import Window +from kivy.properties import BooleanProperty from ..log import get_logger from .pager import PagerView @@ -69,8 +71,10 @@ _kv = (f''' size: self.texture_size # color: {colorcode('gray')} # font_color: {colorcode('gray')} - font_name: 'Roboto-Regular' - background_color: [0]*4 # by default transparent; use row color + # font_name: 'Hack-Regular' + # by default transparent; use row color + # if `highlight` is set use i3 + background_color: {_i3_rgba} if self.click_toggle else [0]*4 # background_color: {_cell_rgba} # spacing: 0, 0 # padding: [0]*4 @@ -148,6 +152,8 @@ class Cell(Button): ``key`` is the column name index value. """ + click_toggle = BooleanProperty(False) + def __init__(self, key=None, is_header=False, **kwargs): super(Cell, self).__init__(**kwargs) self.key = key @@ -177,7 +183,7 @@ class HeaderCell(Cell): # mark this cell as the last selected table.last_clicked_col_cell = self # sort and render the rows immediately - self.row.table.render_rows(table.quote_cache) + self.row.table.render_rows(table.symbols2rows.values()) # TODO: make this some kind of small geometry instead # (maybe like how trading view does it). @@ -310,6 +316,9 @@ class Row(GridLayout, HoverBehavior): def get_cell(self, key): return self._cell_widgets.get(key) + def get_field(self, key): + return self._last_record[key] + def _append_cell(self, text, key, header=False): if not len(self._cell_widgets) < self.cols: raise ValueError(f"Can not append more then {self.cols} cells") @@ -378,12 +387,10 @@ class TickerTable(GridLayout): super(TickerTable, self).__init__(**kwargs) self.symbols2rows = {} self.sort_key = sort_key - self.quote_cache = {} - self.row_filter = lambda item: item # for tracking last clicked column header cell self.last_clicked_col_cell = None - self._last_row_toggle = 0 - self._rendered = set() + self._symbols2index = {} + self._sorted = [] def append_row(self, key, row): """Append a `Row` of `Cell` objects to this table. @@ -391,36 +398,51 @@ class TickerTable(GridLayout): # store ref to each row self.symbols2rows[key] = row self.add_widget(row) + self._sorted.append(row) return row + def clear(self): + self.clear_widgets() + self._sorted.clear() + def render_rows( self, - pairs: Dict[str, Row], + changed: set, sort_key: str = None, - row_filter=None, ): - """Sort and render all rows on the ticker grid from ``pairs``. + """Sort and render all rows on the ticker grid from ``syms2rows``. """ - self.clear_widgets() - self._rendered.clear() sort_key = sort_key or self.sort_key - # TODO: intead of constantly re-rendering on every - # change do a binary search insert using ``bisect.insort()`` - for row in filter( - row_filter or self.row_filter, - reversed( - sorted( - pairs.values(), - key=lambda row: row._last_record[sort_key] - ) - ) - ): - widget = row.widget - if widget not in self._rendered: - self.add_widget(widget) # row append - self._rendered.add(widget) + key_row_pairs = list(sorted( + [(row.get_field(sort_key), row) for row in self._sorted], + key=lambda item: item[0], + )) + if key_row_pairs: + sorted_keys, sorted_rows = zip(*key_row_pairs) + sorted_keys, sorted_rows = list(sorted_keys), list(sorted_rows) + else: + sorted_keys, sorted_rows = [], [] + + # now remove and re-insert any rows that need to be shuffled + # due to new a new field change + for row in changed: + try: + old_index = sorted_rows.index(row) + except ValueError: + # row is not yet added so nothing to remove + pass else: - log.debug(f"Skipping adding widget {widget}") + del sorted_rows[old_index] + del sorted_keys[old_index] + self._sorted.remove(row) + self.remove_widget(row) + + for row in changed: + key = row.get_field(sort_key) + index = bisect(sorted_keys, key) + sorted_keys.insert(index, key) + self._sorted.insert(index, row) + self.add_widget(row, index=index) def ticker_search(self, patt): """Return sequence of matches when pattern ``patt`` is in a @@ -514,36 +536,41 @@ async def update_quotes( # revert flash state momentarily nursery.start_soon(revert_cells_color, unflash) - cache = {} - table.quote_cache = cache - # initial coloring + to_sort = set() for sym, quote in first_quotes.items(): row = table.get_row(sym) record, displayable = formatter( quote, symbol_data=symbol_data) row.update(record, displayable) color_row(row, record, {}) - cache[sym] = row + to_sort.add(row.widget) + + table.render_rows(to_sort) log.debug("Finished initializing update loop") task_status.started() # real-time cell update loop async for quotes in agen: # new quotes data only + to_sort = set() for symbol, quote in quotes.items(): record, displayable = formatter( quote, symbol_data=symbol_data) row = table.get_row(symbol) cells = row.update(record, displayable) color_row(row, record, cells) - cache[symbol] = row - table.render_rows(cache) + if table.sort_key in record: + to_sort.add(row.widget) + + if to_sort: + table.render_rows(to_sort) + log.debug("Waiting on quotes") log.warn("Data feed connection dropped") # XXX: if we're cancelled this should never get called - nursery.cancel_scope.cancel() + # nursery.cancel_scope.cancel() async def _async_main( @@ -638,36 +665,37 @@ async def _async_main( # set up a pager view for large ticker lists table.bind(minimum_height=table.setter('height')) - async with trio.open_nursery() as nursery: - pager = PagerView( - container=box, - contained=table, - nursery=nursery - ) - box.add_widget(pager) + try: + async with trio.open_nursery() as nursery: + pager = PagerView( + container=box, + contained=table, + nursery=nursery + ) + box.add_widget(pager) - widgets = { - 'root': box, - 'table': table, - 'box': box, - 'header': header, - 'pager': pager, - } - nursery.start_soon( - update_quotes, - nursery, - brokermod.format_stock_quote, - widgets, - quote_gen, - sd, - quotes - ) + widgets = { + 'root': box, + 'table': table, + 'box': box, + 'header': header, + 'pager': pager, + } + nursery.start_soon( + update_quotes, + nursery, + brokermod.format_stock_quote, + widgets, + quote_gen, + sd, + quotes + ) - try: # Trio-kivy entry point. await async_runTouchApp(widgets['root']) # run kivy - finally: - # cancel aysnc gen call - await quote_gen.aclose() # cancel GUI update task nursery.cancel_scope.cancel() + finally: + with trio.open_cancel_scope(shield=True): + # cancel aysnc gen call + await quote_gen.aclose() From 7ed409501d8d830c8f6ba1a5340db12ba875d342 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 15 Dec 2018 16:38:33 -0500 Subject: [PATCH 20/42] Even less bouncy --- piker/ui/pager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/pager.py b/piker/ui/pager.py index 5b38aa76..d70175d8 100644 --- a/piker/ui/pager.py +++ b/piker/ui/pager.py @@ -184,5 +184,5 @@ class PagerView(ScrollView): _, yscale = self.convert_distance_to_scroll(0, pxs) new = self.scroll_y + (yscale * {'u': 1, 'd': -1}[direction]) # bound to near [0, 1] to avoid "over-scrolling" - limited = max(-0.01, min(new, 1.01)) + limited = max(-0.001, min(new, 1.001)) self.scroll_y = limited From a13b13e144b503e2e75d2829e31af643cad14dcf Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 15 Dec 2018 19:40:54 -0500 Subject: [PATCH 21/42] Highlight current expiry; mutex data feed access --- piker/ui/option_chain.py | 109 ++++++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 42 deletions(-) diff --git a/piker/ui/option_chain.py b/piker/ui/option_chain.py index 9a8b4d74..d9d9bab6 100644 --- a/piker/ui/option_chain.py +++ b/piker/ui/option_chain.py @@ -6,7 +6,6 @@ Launch with ``piker options ``. import types from functools import partial from typing import Dict, List -# import typing import trio from async_generator import asynccontextmanager @@ -30,7 +29,7 @@ async def modify_symbol(symbol): pass -class StrikeCell(Cell): +class StrikeCell(HeaderCell): """Strike cell""" @@ -61,9 +60,6 @@ class StrikeRow(BoxLayout): table=None, **kwargs, ) -> None: - # if self.is_populated(): - # raise TypeError(f"{self} can only append two sub-rows?") - # the 'contract_type' determines whether this # is a put or call row contract_type = record['contract_type'] @@ -75,6 +71,7 @@ class StrikeRow(BoxLayout): # reverse order of call side cells if contract_type == 'call': record = dict(list(reversed(list(record.items())))) + row = Row( record, bidasks=bidasks, @@ -99,7 +96,7 @@ class StrikeRow(BoxLayout): self.strike, StrikeCell( key=self.strike, text=str(self.strike), - is_header=True, + # is_header=True, # make centre strike cell nice and small size_hint=(1/10., 1), ) @@ -122,10 +119,25 @@ class StrikeRow(BoxLayout): self._sub_rows[record['contract_type']].update( record, displayable) + def get_field(self, key): + """Always sort on the lone field, the strike price. + """ + return int(self.strike) + + +class ExpiryButton(Cell): + # must be set to allow 'plain bg colors' since default texture is grey + background_normal = '' -class ExpiryButton(HeaderCell): def on_press(self, value=None): + # import pdb; pdb.set_trace() + last = self.chain._last_expiry + if last: + last.click_toggle = False + self.chain._last_expiry = self + log.info(f"Clicked {self}") + self.click_toggle = True if self.chain.sub[1] == self.key: log.info(f"Clicked {self} is already selected") return @@ -142,39 +154,50 @@ class DataFeed(object): self.brokermod = brokermod self.sub = None self.quote_gen = None + self._mutex = trio.StrictFIFOLock() async def open_stream(self, symbols, rate=3, test=None): - if self.quote_gen is not None and symbols != self.sub: - log.info(f"Stopping existing subscription for {self.sub}") - await self.quote_gen.aclose() - self.sub = symbols + async with self._mutex: + try: + if self.quote_gen is not None and symbols != self.sub: + log.info( + f"Stopping pre-existing subscription for {self.sub}") + await self.quote_gen.aclose() + self.sub = symbols - if test: - # stream from a local test file - quote_gen = await self.portal.run( - "piker.brokers.data", 'stream_from_file', - filename=test - ) - else: - # start live streaming from broker daemon - quote_gen = await self.portal.run( - "piker.brokers.data", - 'start_quote_stream', - broker=self.brokermod.name, - symbols=symbols, - feed_type='option', - rate=rate, - ) + if test: + # stream from a local test file + quote_gen = await self.portal.run( + "piker.brokers.data", 'stream_from_file', + filename=test + ) + else: + log.info(f"Starting new stream for {self.sub}") + # start live streaming from broker daemon + quote_gen = await self.portal.run( + "piker.brokers.data", + 'start_quote_stream', + broker=self.brokermod.name, + symbols=symbols, + feed_type='option', + rate=rate, + ) - # get first quotes response - log.debug(f"Waiting on first quote for {symbols}...") - quotes = await quote_gen.__anext__() + # get first quotes response + log.debug(f"Waiting on first quote for {symbols}...") + quotes = {} + with trio.move_on_after(5): + quotes = await quote_gen.__anext__() - self.quote_gen = quote_gen - self.first_quotes = quotes - # self.records = records - # self.displayables = displayables - return quote_gen, quotes + self.quote_gen = quote_gen + self.first_quotes = quotes + # self.records = records + # self.displayables = displayables + return quote_gen, quotes + except Exception: + if self.quote_gen: + await self.quote_gen.aclose() + raise class OptionChain(object): @@ -194,11 +217,13 @@ class OptionChain(object): self.bidasks = bidasks self._strikes2rows = {} self._nursery = None + self._update_nursery = None self.feed = feed self._update_cs = None # TODO: this should be moved down to the data feed layer # right now it's only needed for the UI uupdate loop to cancel itself self._first_quotes = None + self._last_expiry = None @asynccontextmanager async def open_scope(self): @@ -215,6 +240,7 @@ class OptionChain(object): self._nursery = n n.start_soon(self.start_updating) yield self + n.cancel_scope.cancel() self._nursery = None await self.feed.quote_gen.aclose() @@ -223,9 +249,8 @@ class OptionChain(object): """Clear the strike rows from the internal table. """ table = self.widgets['table'] - table.clear_widgets() - for strike in self._strikes2rows.copy(): - self._strikes2rows.pop(strike) + table.clear() + self._strikes2rows.clear() def render_rows(self, records, displayables): """Render all strike rows in the internal table. @@ -255,8 +280,7 @@ class OptionChain(object): table.symbols2rows[symbol] = row if strike not in self._strikes2rows: - # readding widgets is an error - table.add_widget(strike_row) + # re-adding widgets is an error self._strikes2rows[strike] = strike_row log.debug("Finished rendering rows!") @@ -278,9 +302,11 @@ class OptionChain(object): async def start_updating(self): if self._update_cs: + log.warn("Cancelling existing update task") self._update_cs.cancel() await trio.sleep(0) + # drop all current rows self.clear() if self._nursery is None: @@ -301,11 +327,10 @@ class OptionChain(object): with trio.open_cancel_scope() as cs: self._update_cs = cs - # start quote update loop await n.start( partial( update_quotes, - self._nursery, + n, self.feed.brokermod.format_option_quote, self.widgets, quote_gen, From 1f608b2498a08ff146ba0e961edf726e602db091 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 15 Dec 2018 19:41:12 -0500 Subject: [PATCH 22/42] Even less latent --- piker/ui/kivy/mouse_over.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/piker/ui/kivy/mouse_over.py b/piker/ui/kivy/mouse_over.py index 8b472ce7..c9ad149b 100644 --- a/piker/ui/kivy/mouse_over.py +++ b/piker/ui/kivy/mouse_over.py @@ -97,9 +97,8 @@ class MouseOverBehavior(object): MouseOverBehavior.remove(self) @classmethod - # try throttling to 1ms latency (doesn't seem to work - # best I can get is 0.01 ?) - @triggered(timeout=0.015, interval=False) + # throttle at 10ms latency + @triggered(timeout=0.01, interval=False) def _on_mouse_pos(cls, *args): log.debug(f"{cls} time since last call: {time.time() - cls._last_time}") cls._last_time = time.time() From 3fd01c42f27ac47ab44a6f95e5419f8a928f53b1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 15 Dec 2018 19:42:15 -0500 Subject: [PATCH 23/42] Define highlight on click logic in `Cell` --- piker/ui/monitor.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/piker/ui/monitor.py b/piker/ui/monitor.py index be90f31b..d57a2d00 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/monitor.py @@ -64,25 +64,29 @@ _kv = (f''' font_size: 21 + # make text wrap to botom text_size: self.size halign: 'center' valign: 'middle' size: self.texture_size + + # don't set these as the update loop already does it # color: {colorcode('gray')} # font_color: {colorcode('gray')} # font_name: 'Hack-Regular' - # by default transparent; use row color - # if `highlight` is set use i3 - background_color: {_i3_rgba} if self.click_toggle else [0]*4 - # background_color: {_cell_rgba} + + # if `highlight` is set use i3 color by default transparent; use row color + # this is currently used for expiry cells on the options chain + background_color: {_i3_rgba} if self.click_toggle else {_black_rgba} + # must be set to allow 'plain bg colors' since default texture is grey + # background_normal: '' # spacing: 0, 0 - # padding: [0]*4 + # padding: 3, 3 + font_size: 21 - background_color: [0]*4 # by default transparent; use row color - # background_color: {_cell_rgba} # canvas.before: # Color: # rgba: [0.13]*4 @@ -93,6 +97,7 @@ _kv = (f''' # # border: [0, {_bs} , 0, {_bs}] # border: [0, {_bs} , 0, 0] + spacing: [{_bs}] # row_force_default: True @@ -102,15 +107,16 @@ _kv = (f''' Color: # i3 style gray as background rgba: {_i3_rgba} - # rgba: {_cell_rgba} Rectangle: # scale with container self here refers to the widget i.e BoxLayout pos: self.pos size: self.size + spacing: [{_bs}, 0] + # minimum_height: 200 # should be pulled from Cell text size # minimum_width: 200 @@ -120,7 +126,6 @@ _kv = (f''' spacing: [0] canvas.before: Color: - # rgba: [0]*4 rgba: {_cell_rgba} Rectangle: # self here refers to the widget i.e Row(GridLayout) @@ -136,7 +141,6 @@ _kv = (f''' # radius: (0,) - # part of the `PagerView` size_hint: 1, None From 5af90c044f51c7bc890b90440ee8cd784316c739 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 16 Dec 2018 23:52:10 -0500 Subject: [PATCH 24/42] Drop contracts cache; that wasn't the bottleneck This also fixes a bug where option subscriptions weren't actually being changed when a new call was made.. --- piker/brokers/questrade.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 7fa00279..fc75ff56 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -5,7 +5,6 @@ import time from datetime import datetime from functools import partial import configparser -from operator import itemgetter from typing import List, Tuple, Dict, Any, Iterator, NamedTuple import trio @@ -418,9 +417,8 @@ async def get_client() -> Client: log.debug("Check time to ensure access token is valid") try: # await client.api.time() - quote = await client.quote(['RY.TO']) - except Exception as err: - # import pdb; pdb.set_trace() + await client.quote(['RY.TO']) + except Exception: # access token is likely no good log.warn(f"Access token {client.access_data['access_token']} seems" f" expired, forcing refresh") @@ -501,18 +499,15 @@ async def option_quoter(client: Client, tickers: List[str]): @async_lifo_cache(maxsize=128) async def get_contract_by_date( sym_date_pairs: Tuple[Tuple[str, str]], - _contract_cache: dict = {} ): """For each tuple, ``(symbol_date_1, symbol_date_2, ... , symbol_date_n)`` return a contract dict. """ symbols, dates = zip(*sym_date_pairs) - if not _contract_cache: - contracts = await client.get_all_contracts(symbols) - _contract_cache.update(contracts) + contracts = await client.get_all_contracts(symbols) selected = {} - for key, val in _contract_cache.items(): + for key, val in contracts.items(): if key.expiry in dates: selected[key] = val From d4e36b1e55d4e9e774b1aed6b4968102bfaefbe1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 17 Dec 2018 19:15:29 -0500 Subject: [PATCH 25/42] Jeeze, don't overwrite the payload for each channel... --- piker/brokers/data.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/piker/brokers/data.py b/piker/brokers/data.py index e449a770..b076ef9c 100644 --- a/piker/brokers/data.py +++ b/piker/brokers/data.py @@ -158,17 +158,14 @@ async def fan_out_to_chans( diff_cached=diff_cached, ): chan_payloads = {} - payload = {} for quote in quotes: - # is this too QT specific? - symbol = quote['symbol'] - payload[symbol] = quote # set symbol quotes for each subscriber - for chan, cid in symbols2chans.get(quote['key'], set()): + # for chan, cid in symbols2chans.get(quote['key'], set()): + for chan, cid in symbols2chans[quote['key']]: chan_payloads.setdefault( chan, - {'yield': payload, 'cid': cid} - ) + {'yield': {}, 'cid': cid} + )['yield'].update({quote['symbol']: quote}) # deliver to each subscriber (fan out) if chan_payloads: From 11222e11763b68b64ecbf93b9624d35cd962d960 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 18 Dec 2018 20:28:26 -0500 Subject: [PATCH 26/42] Only resort when the sort field actually changed --- piker/ui/monitor.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/piker/ui/monitor.py b/piker/ui/monitor.py index d57a2d00..be5915a0 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/monitor.py @@ -7,7 +7,7 @@ Launch with ``piker monitor ``. """ from itertools import chain from types import ModuleType, AsyncGeneratorType -from typing import List, Callable, Dict +from typing import List, Callable from bisect import bisect import trio @@ -21,6 +21,7 @@ from kivy import utils from kivy.app import async_runTouchApp from kivy.core.window import Window from kivy.properties import BooleanProperty +from kivy.uix.behaviors import ButtonBehavior from ..log import get_logger from .pager import PagerView @@ -161,8 +162,12 @@ class Cell(Button): def __init__(self, key=None, is_header=False, **kwargs): super(Cell, self).__init__(**kwargs) self.key = key + self.row = None self.is_header = is_header + # def on_press(self, value=None): + # self.row.on_press() + class HeaderCell(Cell): """Column header cell label. @@ -257,7 +262,7 @@ class BidAskLayout(StackLayout): return [self.last, self.bid, self.ask] -class Row(GridLayout, HoverBehavior): +class Row(ButtonBehavior, HoverBehavior, GridLayout): """A grid for displaying a row of ticker quote data. """ def __init__( @@ -271,7 +276,7 @@ class Row(GridLayout, HoverBehavior): cell_type=None, **kwargs ): - super(Row, self).__init__(cols=len(record), **kwargs) + super().__init__(cols=len(record), **kwargs) self._cell_widgets = {} self._last_record = record self.table = table @@ -383,16 +388,20 @@ class Row(GridLayout, HoverBehavior): log.debug( f"Left row {self} through {self.border_point}") + def on_press(self, value=None): + log.info(f"Pressed row for {self._last_record['symbol']}") + class TickerTable(GridLayout): """A grid for displaying ticker quote records as a table. """ - def __init__(self, sort_key='%', **kwargs): + def __init__(self, sort_key='%', auto_sort=True, **kwargs): super(TickerTable, self).__init__(**kwargs) self.symbols2rows = {} self.sort_key = sort_key # for tracking last clicked column header cell self.last_clicked_col_cell = None + self._auto_sort = auto_sort self._symbols2index = {} self._sorted = [] @@ -558,15 +567,21 @@ async def update_quotes( async for quotes in agen: # new quotes data only to_sort = set() for symbol, quote in quotes.items(): + row = table.get_row(symbol) record, displayable = formatter( quote, symbol_data=symbol_data) - row = table.get_row(symbol) + + # determine if sorting should happen + sort_key = table.sort_key + new = record[sort_key] + last = row.get_field(sort_key) + if new != last: + to_sort.add(row.widget) + + # update and color cells = row.update(record, displayable) color_row(row, record, cells) - if table.sort_key in record: - to_sort.add(row.widget) - if to_sort: table.render_rows(to_sort) From eb8c9e1a99d27e9eb230e40510b6b579544b5704 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 23 Dec 2018 20:48:06 -0500 Subject: [PATCH 27/42] Symbol subs must be cid specific --- piker/brokers/data.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/piker/brokers/data.py b/piker/brokers/data.py index b076ef9c..f5cff4a8 100644 --- a/piker/brokers/data.py +++ b/piker/brokers/data.py @@ -159,17 +159,16 @@ async def fan_out_to_chans( ): chan_payloads = {} for quote in quotes: - # set symbol quotes for each subscriber - # for chan, cid in symbols2chans.get(quote['key'], set()): - for chan, cid in symbols2chans[quote['key']]: + packet = {quote['symbol']: quote} + for chan, cid in symbols2chans.get(quote['key'], set()): chan_payloads.setdefault( - chan, + (chan, cid), {'yield': {}, 'cid': cid} - )['yield'].update({quote['symbol']: quote}) + )['yield'].update(packet) # deliver to each subscriber (fan out) if chan_payloads: - for chan, payload in chan_payloads.items(): + for (chan, cid), payload in chan_payloads.items(): try: await chan.send(payload) except ( @@ -269,7 +268,7 @@ def modify_quote_stream(broker, feed_type, symbols, chan, cid): chanset = symbols2chans.get(ticker) # XXX: cid will be different on unsub call for item in chanset.copy(): - if chan in item: + if (chan, cid) == item: chanset.discard(item) if not chanset: From de4fab873bef0e9950b0d05651f59731ca7be9d9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 23 Dec 2018 21:25:56 -0500 Subject: [PATCH 28/42] Ids should be allowed without contracts --- piker/brokers/questrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index fc75ff56..aeed571c 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -92,7 +92,7 @@ class _API: async def option_quotes( self, - contracts: Dict[ContractsKey, Dict[int, dict]], + contracts: Dict[ContractsKey, Dict[int, dict]] = {}, option_ids: List[int] = [], # if you don't want them all ) -> dict: """Retrieve option chain quotes for all option ids or by filter(s). From a7fb55179c977cbe41a47d1fe269124cfffbafb3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 23 Dec 2018 21:26:57 -0500 Subject: [PATCH 29/42] Handle weekend data from QT yet again --- piker/ui/monitor.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/piker/ui/monitor.py b/piker/ui/monitor.py index be5915a0..1ff98d0b 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/monitor.py @@ -81,6 +81,7 @@ _kv = (f''' # this is currently used for expiry cells on the options chain background_color: {_i3_rgba} if self.click_toggle else {_black_rgba} # must be set to allow 'plain bg colors' since default texture is grey + # but right now is only set for option chain expiry buttons # background_normal: '' # spacing: 0, 0 # padding: 3, 3 @@ -165,8 +166,8 @@ class Cell(Button): self.row = None self.is_header = is_header - # def on_press(self, value=None): - # self.row.on_press() + def on_press(self, value=None): + self.row.on_press() class HeaderCell(Cell): @@ -352,13 +353,16 @@ class Row(ButtonBehavior, HoverBehavior, GridLayout): fgreen = colorcode('forestgreen') red = colorcode('red2') for key, val in record.items(): - # logic for cell text coloring: up-green, down-red - if self._last_record[key] < val: - color = fgreen - elif self._last_record[key] > val: - color = red - else: - color = gray + last = self.get_field(key) + color = gray + try: + # logic for cell text coloring: up-green, down-red + if last < val: + color = fgreen + elif last > val: + color = red + except TypeError: + log.warn(f"wtf QT {val} is not regular?") cell = self.get_cell(key) # some displayable fields might have specifically From 6cc8b4cc2fa89172bab5cbbd757ae6771b97f020 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 23 Dec 2018 21:27:47 -0500 Subject: [PATCH 30/42] Test duplicate feed type quoting --- tests/test_questrade.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/tests/test_questrade.py b/tests/test_questrade.py index 39b64cc7..6730884d 100644 --- a/tests/test_questrade.py +++ b/tests/test_questrade.py @@ -186,7 +186,7 @@ async def stream_option_chain(portal, symbols): ``symbols`` arg is ignored here. """ - symbol = 'APHA.TO' # your fave greenhouse LP + symbol = symbols[0] async with qt.get_client() as client: contracts = await client.get_all_contracts([symbol]) @@ -204,14 +204,18 @@ async def stream_option_chain(portal, symbols): rate=4, diff_cached=False, ) + # latency arithmetic + loops = 8 + rate = 1/3. # 3 rps + timeout = loops / rate + try: # wait on the data streamer to actually start # delivering await agen.__anext__() # it'd sure be nice to have an asyncitertools here... - with trio.fail_after(2.1): - loops = 8 + with trio.fail_after(timeout): count = 0 async for quotes in agen: # print(f'got quotes for {quotes.keys()}') @@ -241,8 +245,7 @@ async def stream_option_chain(portal, symbols): ) await agen.__anext__() - with trio.fail_after(2.1): - loops = 8 + with trio.fail_after(timeout): count = 0 async for quotes in agen: for symbol, quote in quotes.items(): @@ -263,6 +266,7 @@ async def stream_stocks(portal, symbols): 'start_quote_stream', broker='questrade', symbols=symbols, + diff_cached=False, ) try: # it'd sure be nice to have an asyncitertools here... @@ -282,8 +286,14 @@ async def stream_stocks(portal, symbols): (stream_stocks,), (stream_option_chain,), (stream_stocks, stream_option_chain), + (stream_stocks, stream_stocks), + (stream_option_chain, stream_option_chain), + ], + ids=[ + 'stocks', 'options', + 'stocks_and_options', 'stocks_and_stocks', + 'options_and_options', ], - ids=['stocks', 'options', 'stocks_and_options'], ) @tractor_test async def test_quote_streaming(tmx_symbols, loglevel, stream_what): @@ -301,9 +311,16 @@ async def test_quote_streaming(tmx_symbols, loglevel, stream_what): 'piker.brokers.core' ], ) + if len(stream_what) > 1: + # stream disparate symbol sets per task + first, *tail = tmx_symbols + symbols = ([first], tail) + else: + symbols = [tmx_symbols] + async with trio.open_nursery() as n: - for func in stream_what: - n.start_soon(func, portal, tmx_symbols) + for syms, func in zip(symbols, stream_what): + n.start_soon(func, portal, syms) # stop all spawned subactors await nursery.cancel() From fb876f3770cfefaa5a754f3bdee29cbacbfcdb28 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 25 Dec 2018 12:38:04 -0500 Subject: [PATCH 31/42] Drop `OptionChain.start_feed()` --- piker/ui/option_chain.py | 119 ++++++++++++++++++++------------------- 1 file changed, 61 insertions(+), 58 deletions(-) diff --git a/piker/ui/option_chain.py b/piker/ui/option_chain.py index d9d9bab6..00e5cf5a 100644 --- a/piker/ui/option_chain.py +++ b/piker/ui/option_chain.py @@ -124,13 +124,15 @@ class StrikeRow(BoxLayout): """ return int(self.strike) + def rowsitems(self): + return self._sub_rows.items() + class ExpiryButton(Cell): # must be set to allow 'plain bg colors' since default texture is grey background_normal = '' def on_press(self, value=None): - # import pdb; pdb.set_trace() last = self.chain._last_expiry if last: last.click_toggle = False @@ -146,24 +148,24 @@ class ExpiryButton(Cell): class DataFeed(object): - """Data feed client for streaming symbol data from a remote - broker data source. + """Data feed client for streaming symbol data from a (remote) + ``brokerd`` data daemon. """ def __init__(self, portal, brokermod): self.portal = portal self.brokermod = brokermod - self.sub = None + self._symbols = None self.quote_gen = None self._mutex = trio.StrictFIFOLock() - async def open_stream(self, symbols, rate=3, test=None): + async def open_stream(self, symbols, rate=1, test=None): async with self._mutex: try: - if self.quote_gen is not None and symbols != self.sub: + if self.quote_gen is not None and symbols != self._symbols: log.info( - f"Stopping pre-existing subscription for {self.sub}") + f"Stopping existing subscription for {self._symbols}") await self.quote_gen.aclose() - self.sub = symbols + self._symbols = symbols if test: # stream from a local test file @@ -172,7 +174,7 @@ class DataFeed(object): filename=test ) else: - log.info(f"Starting new stream for {self.sub}") + log.info(f"Starting new stream for {self._symbols}") # start live streaming from broker daemon quote_gen = await self.portal.run( "piker.brokers.data", @@ -203,6 +205,8 @@ class DataFeed(object): class OptionChain(object): """A real-time options chain UI. """ + _title = "option chain: {symbol}\t(press ? for help)" + def __init__( self, symbol: str, @@ -210,7 +214,7 @@ class OptionChain(object): widgets: dict, bidasks: Dict[str, List[str]], feed: DataFeed, - rate: int = 1, + rate: int, ): self.sub = (symbol, expiry) self.widgets = widgets @@ -220,15 +224,16 @@ class OptionChain(object): self._update_nursery = None self.feed = feed self._update_cs = None + self._quote_gen = None # TODO: this should be moved down to the data feed layer # right now it's only needed for the UI uupdate loop to cancel itself self._first_quotes = None self._last_expiry = None @asynccontextmanager - async def open_scope(self): - """Open an internal resource and update task scope required - to allow for dynamic real-time operation. + async def open_update_scope(self): + """Open an internal update task scope required to allow + for dynamic real-time operation. """ # assign us to each expiry button for key, button in ( @@ -238,20 +243,24 @@ class OptionChain(object): async with trio.open_nursery() as n: self._nursery = n - n.start_soon(self.start_updating) + n.start_soon(self._start_displaying, *self.sub) yield self n.cancel_scope.cancel() self._nursery = None + # make sure we always tear down our existing data feed await self.feed.quote_gen.aclose() - def clear(self): + def clear_strikes(self): """Clear the strike rows from the internal table. """ table = self.widgets['table'] table.clear() self._strikes2rows.clear() + def clear_expiries(self): + pass + def render_rows(self, records, displayables): """Render all strike rows in the internal table. """ @@ -275,7 +284,7 @@ class OptionChain(object): # using each contracts "symbol" so that the quote updater # task can look up the right row to update easily # See update_quotes() and ``Row`` internals for details. - for contract_type, row in strike_row._sub_rows.items(): + for contract_type, row in strike_row.rowsitems(): symbol = row._last_record['symbol'] table.symbols2rows[symbol] = row @@ -285,38 +294,39 @@ class OptionChain(object): log.debug("Finished rendering rows!") - async def start_feed( - self, - symbol: str, - expiry: str, - # max QT rate per API customer is approx 4 rps - # and usually 3 rps is allocated to the stock monitor - rate: int = 1, - test: str = None - ): - if self.feed.sub != self.sub: - return await self.feed.open_stream([(symbol, expiry)], rate=rate) - else: - feed = self.feed - return feed.quote_gen, feed.first_quotes + async def _start_displaying(self, symbol, expiry): + """Main routine to start displaying the real time updated strike + table. + + Clear any existing data feed subscription that is no longer needed + (eg. when clicking a new expiry button) spin up a new subscription, + populate the table and start updating it. + """ + # set window title + self.widgets['window'].set_title( + self._title.format(symbol=symbol) + ) - async def start_updating(self): if self._update_cs: log.warn("Cancelling existing update task") self._update_cs.cancel() await trio.sleep(0) - # drop all current rows - self.clear() + if self._quote_gen: + await self._quote_gen.aclose() + + self.clear_strikes() if self._nursery is None: raise RuntimeError( - "You must call await `start()` first!") + "You must call open this chain's update scope first!") n = self._nursery - log.debug(f"Waiting on first_quotes for {self.sub}") - quote_gen, first_quotes = await self.start_feed(*self.sub) - log.debug(f"Got first_quotes for {self.sub}") + log.debug(f"Waiting on first_quotes for {symbol}:{expiry}") + self._quote_gen, first_quotes = await self.feed.open_stream( + [(symbol, expiry)] + ) + log.debug(f"Got first_quotes for {symbol}:{expiry}") # redraw the UI records, displayables = zip(*[ @@ -333,7 +343,7 @@ class OptionChain(object): n, self.feed.brokermod.format_option_quote, self.widgets, - quote_gen, + self._quote_gen, symbol_data={}, first_quotes=first_quotes, ) @@ -341,21 +351,26 @@ class OptionChain(object): def start_displaying(self, symbol, expiry): self.sub = (symbol, expiry) - self._nursery.start_soon(self.start_updating) + self._nursery.start_soon(self._start_displaying, symbol, expiry) async def new_chain_ui( portal: tractor._portal.Portal, symbol: str, - expiry: str, - contracts, brokermod: types.ModuleType, nursery: trio._core._run.Nursery, - rate: int = 1, + rate: int = 2, ) -> None: """Create and return a new option chain UI. """ + # retreive all contracts just because we need a default when the + # UI starts up + all_contracts = await contracts(brokermod, symbol) + # start streaming soonest contract by default + expiry = next(iter(all_contracts)).expiry + widgets = {} + # define bid-ask "stacked" cells # (TODO: needs some rethinking and renaming for sure) bidasks = brokermod._option_bidasks @@ -369,17 +384,12 @@ async def new_chain_ui( feed, rate=rate, ) - - quote_gen, first_quotes = await chain.start_feed(symbol, expiry) + quote_gen, first_quotes = await chain.feed.open_stream([chain.sub]) records, displayables = zip(*[ brokermod.format_option_quote(quote, {}) for quote in first_quotes.values() ]) - # build out root UI - title = f"option chain: {symbol}\t(press ? for help)" - Window.set_title(title) - # use `monitor` styling for now from .monitor import _kv Builder.load_string(_kv) @@ -390,7 +400,7 @@ async def new_chain_ui( # TODO: figure out how to compact these buttons expiries = { key.expiry: key.expiry[:key.expiry.find('T')] - for key in contracts + for key in all_contracts } expiry_buttons = Row( record=expiries, @@ -439,6 +449,7 @@ async def new_chain_ui( ) container.add_widget(pager) widgets.update({ + 'window': Window, 'root': container, 'container': container, 'table': table, @@ -459,24 +470,16 @@ async def _async_main( This is started with cli cmd `piker options`. ''' - # retreive all contracts just because we need a default when the - # UI starts up - all_contracts = await contracts(brokermod, symbol) - # start streaming soonest contract by default - first_expiry = next(iter(all_contracts)).expiry - async with trio.open_nursery() as nursery: # set up a pager view for large ticker lists chain = await new_chain_ui( portal, symbol, - first_expiry, - all_contracts, brokermod, nursery, rate=rate, ) - async with chain.open_scope(): + async with chain.open_update_scope(): try: # Trio-kivy entry point. await async_runTouchApp(chain.widgets['root']) # run kivy From dc581d0bdcc0601cd8da79d36bf3dcc24fd6eada Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 26 Dec 2018 13:30:50 -0500 Subject: [PATCH 32/42] Handle "adjusted contract" chains per root --- piker/brokers/questrade.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index aeed571c..fca579a4 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -315,17 +315,28 @@ class Client: contracts.items(), key=lambda item: item[0].expiry ): - by_key[ - ContractsKey( - key.symbol, - key.id, - # converting back - maybe just do this initially? - key.expiry.isoformat(timespec='microseconds'), - ) - ] = { - item['strikePrice']: item for item in - byroot['chainPerRoot'][0]['chainPerStrikePrice'] - } + for chain in byroot['chainPerRoot']: + optroot = chain['optionRoot'] + suffix = '' + + # handle QTs "adjusted contracts" (aka adjusted for + # the underlying in some way; usually has a '(1)' in + # the expiry key in their UI) + adjusted_contracts = optroot != key.symbol + if adjusted_contracts: + suffix = '(' + optroot[len(key.symbol):] + ')' + + by_key[ + ContractsKey( + key.symbol + suffix, + key.id, + # converting back - maybe just do this initially? + key.expiry.isoformat(timespec='microseconds'), + ) + ] = { + item['strikePrice']: item for item in + chain['chainPerStrikePrice'] + } # fill out contract id to strike expiry map for tup, bystrikes in by_key.items(): From 1866dd181294b9c6cda3d3873bacb882df5a1c9d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 29 Dec 2018 15:44:32 -0500 Subject: [PATCH 33/42] Fix for adjusted contracts subscription bug If quotes are pushed using the adjusted contract symbol (i.e. with trailing '-1' suffix) the subscriber won't receive them under the normal symbol. The logic was wrong for determining whether to add a suffix (was failing for any symbol with an exchange suffix) which was causing normal data feed subscriptions to fail to match in every case. I did some testing of the `optionsIds` parameter to the option quote endpoint and found that it limits you to 100 symbols so it's not practical for real-time "all-strike"" chain updating; we have to stick to filters for now. The only real downside of this is that it seems multiple filters across multiple symbols is quite latent. I need to toy with it more to be sure it's not something slow on the client side. Oh, and store option contract to ids in a `dict` for now as we may want to try the `optionsIds` thing again down the road as I coordinate with the QT tech team. --- piker/brokers/questrade.py | 66 +++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index fca579a4..2a588562 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -107,6 +107,8 @@ class _API: ] resp = await self._sess.post( path=f'/markets/quotes/options', + # XXX: b'{"code":1024,"message":"The size of the array requested is not valid: optionIds"}' + # ^ what I get when trying to use too many ids manually... json={'filters': filters, 'optionIds': option_ids} ) return resproc(resp, log)['optionQuotes'] @@ -125,8 +127,8 @@ class Client: self.access_data = {} self._reload_config(config) self._symbol_cache: Dict[str, int] = {} - self._contracts2expiries = {} self._optids2contractinfo = {} + self._contract2ids = {} def _reload_config(self, config=None, **kwargs): log.warn("Reloading access config data") @@ -317,14 +319,13 @@ class Client: ): for chain in byroot['chainPerRoot']: optroot = chain['optionRoot'] - suffix = '' # handle QTs "adjusted contracts" (aka adjusted for # the underlying in some way; usually has a '(1)' in # the expiry key in their UI) - adjusted_contracts = optroot != key.symbol - if adjusted_contracts: - suffix = '(' + optroot[len(key.symbol):] + ')' + adjusted_contracts = optroot not in key.symbol + tail = optroot[len(key.symbol):] + suffix = '-' + tail if adjusted_contracts else '' by_key[ ContractsKey( @@ -344,12 +345,16 @@ class Client: for key, contract_type in ( ('callSymbolId', 'call'), ('putSymbolId', 'put') ): - self._optids2contractinfo[ - ids[key]] = { - 'strike': strike, - 'expiry': tup.expiry, - 'contract_type': contract_type, - } + contract_int_id = ids[key] + self._optids2contractinfo[contract_int_id] = { + 'strike': strike, + 'expiry': tup.expiry, + 'contract_type': contract_type, + 'contract_key': tup, + } + # store ids per contract + self._contract2ids.setdefault( + tup, set()).add(contract_int_id) return by_key async def option_chains( @@ -359,22 +364,31 @@ class Client: ) -> Dict[str, Dict[str, Dict[str, Any]]]: """Return option chain snap quote for each ticker in ``symbols``. """ - batch = [] - for key, bystrike in contracts.items(): - quotes = await self.api.option_quotes({key: bystrike}) - for quote in quotes: - # index by .symbol, .expiry since that's what - # a subscriber (currently) sends initially - quote['key'] = (key[0], key[2]) - # update with expiry and strike (Obviously the - # QT api designers are using some kind of severely - # stupid disparate table system where they keep - # contract info in a separate table from the quote format - # keys. I'm really not surprised though - windows shop..) - quote.update(self._optids2contractinfo[quote['symbolId']]) - batch.extend(quotes) + quotes = await self.api.option_quotes(contracts=contracts) + # XXX the below doesn't work so well due to the symbol count + # limit per quote request + # quotes = await self.api.option_quotes(option_ids=list(contract_ids)) + for quote in quotes: + id = quote['symbolId'] + contract_info = self._optids2contractinfo[id].copy() + key = contract_info.pop('contract_key') - return batch + # XXX TODO: this currently doesn't handle adjusted contracts + # (i.e. ones that we stick a '(1)' after) + + # index by .symbol, .expiry since that's what + # a subscriber (currently) sends initially + quote['key'] = (key.symbol, key.expiry) + + # update with expiry and strike (Obviously the + # QT api designers are using some kind of severely + # stupid disparate table system where they keep + # contract info in a separate table from the quote format + # keys. I'm really not surprised though - windows shop..) + # quote.update(self._optids2contractinfo[quote['symbolId']]) + quote.update(contract_info) + + return quotes async def token_refresher(client): From b4fad3f6a9c10605ca4653ab811a9437efa2c2d8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 29 Dec 2018 16:00:18 -0500 Subject: [PATCH 34/42] Logic factoring --- piker/brokers/data.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/piker/brokers/data.py b/piker/brokers/data.py index f5cff4a8..8f321659 100644 --- a/piker/brokers/data.py +++ b/piker/brokers/data.py @@ -147,11 +147,8 @@ async def fan_out_to_chans( """Get quotes for current symbol subscription set. """ symbols = list(symbols2chans.keys()) - if symbols: - # subscription can be changed at any time - return await get_quotes(symbols) - else: - return () + # subscription can be changed at any time + return await get_quotes(symbols) if symbols else () async for quotes in stream_quotes( feed.mod, request, rate, From 3ed750d3241f2feab595a4b0462e95d4fff56d72 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 29 Dec 2018 16:01:07 -0500 Subject: [PATCH 35/42] Add contract table type headers --- piker/ui/option_chain.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/piker/ui/option_chain.py b/piker/ui/option_chain.py index 00e5cf5a..5fae8dbf 100644 --- a/piker/ui/option_chain.py +++ b/piker/ui/option_chain.py @@ -14,6 +14,7 @@ from kivy.uix.boxlayout import BoxLayout from kivy.lang import Builder from kivy.app import async_runTouchApp from kivy.core.window import Window +from kivy.uix.label import Label from ..log import get_logger from ..brokers.core import contracts @@ -188,8 +189,8 @@ class DataFeed(object): # get first quotes response log.debug(f"Waiting on first quote for {symbols}...") quotes = {} - with trio.move_on_after(5): - quotes = await quote_gen.__anext__() + # with trio.move_on_after(5): + quotes = await quote_gen.__anext__() self.quote_gen = quote_gen self.first_quotes = quotes @@ -359,7 +360,7 @@ async def new_chain_ui( symbol: str, brokermod: types.ModuleType, nursery: trio._core._run.Nursery, - rate: int = 2, + rate: int = 1, ) -> None: """Create and return a new option chain UI. """ @@ -412,6 +413,17 @@ async def new_chain_ui( # top row of expiry buttons container.add_widget(expiry_buttons) + # denote calls vs. puts side of table + type_header = BoxLayout( + orientation='horizontal', + size_hint=(1, 1/28.), + ) + calls = Label(text='calls', font_size='20') + puts = Label(text='puts', font_size='20') + type_header.add_widget(calls) + type_header.add_widget(puts) + container.add_widget(type_header) + # figure out header fields for each table based on quote keys headers = displayables[0].keys() header_row = StrikeRow(strike='strike', size_hint=(1, None)) @@ -435,6 +447,8 @@ async def new_chain_ui( size_hint=(1, None), ) container.add_widget(header_row) + + # build out chain tables table = TickerTable( sort_key='strike', cols=1, From 152062ba8a0548835f5feb043688e8618543d81a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 30 Dec 2018 14:59:54 -0500 Subject: [PATCH 36/42] Support pub-sub of monitor's symbol selection --- piker/cli.py | 3 ++- piker/ui/monitor.py | 24 +++++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/piker/cli.py b/piker/cli.py index 8e4db81e..d980dea3 100644 --- a/piker/cli.py +++ b/piker/cli.py @@ -178,8 +178,9 @@ def monitor(loglevel, broker, rate, name, dhost, test, tl): tractor.run( partial(main, tries=1), - name='kivy-monitor', + name='monitor', loglevel=loglevel if tl else None, + rpc_module_paths=['piker.ui.monitor'], ) diff --git a/piker/ui/monitor.py b/piker/ui/monitor.py index 1ff98d0b..1b6d8f3a 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/monitor.py @@ -21,7 +21,6 @@ from kivy import utils from kivy.app import async_runTouchApp from kivy.core.window import Window from kivy.properties import BooleanProperty -from kivy.uix.behaviors import ButtonBehavior from ..log import get_logger from .pager import PagerView @@ -263,7 +262,7 @@ class BidAskLayout(StackLayout): return [self.last, self.bid, self.ask] -class Row(ButtonBehavior, HoverBehavior, GridLayout): +class Row(HoverBehavior, GridLayout): """A grid for displaying a row of ticker quote data. """ def __init__( @@ -383,7 +382,7 @@ class Row(ButtonBehavior, HoverBehavior, GridLayout): log.debug( f"Entered row {self} through {self.border_point}") # don't highlight header row - if getattr(self, 'is_header', None): + if self.is_header: self.hovered = False def on_leave(self): @@ -394,6 +393,9 @@ class Row(ButtonBehavior, HoverBehavior, GridLayout): def on_press(self, value=None): log.info(f"Pressed row for {self._last_record['symbol']}") + if self.table and not self.is_header: + for q in self.table._click_queues: + q.put_nowait(self._last_record['symbol']) class TickerTable(GridLayout): @@ -408,6 +410,7 @@ class TickerTable(GridLayout): self._auto_sort = auto_sort self._symbols2index = {} self._sorted = [] + self._click_queues: List[trio.Queue] = [] def append_row(self, key, row): """Append a `Row` of `Cell` objects to this table. @@ -596,6 +599,21 @@ async def update_quotes( # nursery.cancel_scope.cancel() +async def stream_symbol_selection(): + """An RPC async gen for streaming the symbol corresponding + value corresponding to the last clicked row. + """ + widgets = tractor.current_actor().statespace['widgets'] + table = widgets['table'] + q = trio.Queue(1) + table._click_queues.append(q) + try: + async for symbol in q: + yield symbol + finally: + table._click_queues.remove(q) + + async def _async_main( name: str, portal: tractor._portal.Portal, From 72f417b9c2d4962e51f0da990c5befe84d129129 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 30 Dec 2018 15:00:46 -0500 Subject: [PATCH 37/42] Support monitor linked symbol selection This allows for using a monitor to select the current option chain symbol! The deats: - start a bg task which streams the monitor selected symbol - dynamically repopulate expiry buttons on a newly published symbol - move static widget creation into a chain method to avoid multiple quotes requests at startup - rename a bunch of methods --- piker/ui/option_chain.py | 302 ++++++++++++++++++++++----------------- 1 file changed, 171 insertions(+), 131 deletions(-) diff --git a/piker/ui/option_chain.py b/piker/ui/option_chain.py index 5fae8dbf..86654175 100644 --- a/piker/ui/option_chain.py +++ b/piker/ui/option_chain.py @@ -141,11 +141,7 @@ class ExpiryButton(Cell): log.info(f"Clicked {self}") self.click_toggle = True - if self.chain.sub[1] == self.key: - log.info(f"Clicked {self} is already selected") - return - log.info(f"Subscribing for {self.chain.sub}") - self.chain.start_displaying(self.chain.sub[0], self.key) + self.chain.start_displaying(self.chain.symbol, self.key) class DataFeed(object): @@ -194,14 +190,32 @@ class DataFeed(object): self.quote_gen = quote_gen self.first_quotes = quotes - # self.records = records - # self.displayables = displayables return quote_gen, quotes except Exception: if self.quote_gen: await self.quote_gen.aclose() + self.quote_gen = None raise + def format_quotes(self, quotes): + records, displayables = zip(*[ + self.brokermod.format_option_quote(quote, {}) + for quote in quotes.values() + ]) + return records, displayables + + +@asynccontextmanager +async def find_local_monitor(): + """Establish a portal to a local monitor for triggering + symbol changes. + """ + async with tractor.find_actor('monitor') as portal: + if not portal: + log.warn( + "No monitor app could be found, no symbol link established..") + yield portal + class OptionChain(object): """A real-time options chain UI. @@ -210,14 +224,13 @@ class OptionChain(object): def __init__( self, - symbol: str, - expiry: str, widgets: dict, bidasks: Dict[str, List[str]], feed: DataFeed, rate: int, ): - self.sub = (symbol, expiry) + self.symbol = None + self.expiry = None self.widgets = widgets self.bidasks = bidasks self._strikes2rows = {} @@ -230,21 +243,35 @@ class OptionChain(object): # right now it's only needed for the UI uupdate loop to cancel itself self._first_quotes = None self._last_expiry = None + # flag to determine if one-time widgets have been generated + self._static_widgets_initialized = False + + async def _rx_symbols(self): + async with find_local_monitor() as portal: + if not portal: + log.warn("No local monitor could be found") + return + async for symbol in await portal.run( + 'piker.ui.monitor', + 'stream_symbol_selection', + ): + log.info(f"Changing symbol subscriptions to {symbol}") + self.start_displaying(symbol, self.expiry) @asynccontextmanager - async def open_update_scope(self): + async def open_rt_display(self, nursery, symbol, expiry=None): """Open an internal update task scope required to allow for dynamic real-time operation. """ - # assign us to each expiry button - for key, button in ( - self.widgets['expiry_buttons']._cell_widgets.items() - ): - button.chain = self - + self._parent_nursery = nursery async with trio.open_nursery() as n: self._nursery = n - n.start_soon(self._start_displaying, *self.sub) + # fill out and start updatingn strike table + n.start_soon( + partial(self._start_displaying, symbol, expiry=expiry) + ) + # listen for undlerlying symbol changes from a local monitor app + n.start_soon(self._rx_symbols) yield self n.cancel_scope.cancel() @@ -259,9 +286,6 @@ class OptionChain(object): table.clear() self._strikes2rows.clear() - def clear_expiries(self): - pass - def render_rows(self, records, displayables): """Render all strike rows in the internal table. """ @@ -295,7 +319,7 @@ class OptionChain(object): log.debug("Finished rendering rows!") - async def _start_displaying(self, symbol, expiry): + async def _start_displaying(self, symbol, expiry=None): """Main routine to start displaying the real time updated strike table. @@ -303,10 +327,34 @@ class OptionChain(object): (eg. when clicking a new expiry button) spin up a new subscription, populate the table and start updating it. """ - # set window title - self.widgets['window'].set_title( - self._title.format(symbol=symbol) - ) + # redraw any symbol specific UI components + if self.symbol != symbol or expiry is None: + # set window title + self.widgets['window'].set_title( + self._title.format(symbol=symbol) + ) + + # retreive all contracts to populate expiry row + all_contracts = await contracts(self.feed.brokermod, symbol) + # start streaming soonest contract by default if not provided + expiry = next(iter(all_contracts)).expiry if not expiry else expiry + + # TODO: figure out how to compact these buttons + expiries = { + key.expiry: key.expiry[:key.expiry.find('T')] + for key in all_contracts + } + expiry_row = self.widgets['expiry_row'] + expiry_row.clear_widgets() + + for expiry, justdate in expiries.items(): + button = ExpiryButton(text=str(justdate), key=expiry) + # assign us to each expiry button + button.chain = self + expiry_row.add_widget(button) + + if self.widgets.get('table'): + self.clear_strikes() if self._update_cs: log.warn("Cancelling existing update task") @@ -316,24 +364,23 @@ class OptionChain(object): if self._quote_gen: await self._quote_gen.aclose() - self.clear_strikes() - if self._nursery is None: raise RuntimeError( "You must call open this chain's update scope first!") - n = self._nursery log.debug(f"Waiting on first_quotes for {symbol}:{expiry}") self._quote_gen, first_quotes = await self.feed.open_stream( [(symbol, expiry)] ) log.debug(f"Got first_quotes for {symbol}:{expiry}") + records, displayables = self.feed.format_quotes(first_quotes) - # redraw the UI - records, displayables = zip(*[ - self.feed.brokermod.format_option_quote(quote, {}) - for quote in first_quotes.values() - ]) + # draw static widgets only once + if self._static_widgets_initialized is False: + self._init_static_widgets(displayables) + self._static_widgets_initialized = True + + n = self._nursery self.render_rows(records, displayables) with trio.open_cancel_scope() as cs: @@ -349,10 +396,78 @@ class OptionChain(object): first_quotes=first_quotes, ) ) + self.symbol, self.expiry = symbol, expiry def start_displaying(self, symbol, expiry): - self.sub = (symbol, expiry) - self._nursery.start_soon(self._start_displaying, symbol, expiry) + if self.symbol == symbol and self.expiry == expiry: + log.info(f"Clicked {symbol}:{expiry} is already selected") + return + + log.info(f"Subscribing for {symbol}:{expiry}") + self._nursery.start_soon( + partial(self._start_displaying, symbol, expiry=expiry) + ) + + def _init_static_widgets(self, displayables): + assert self._static_widgets_initialized is False + container = self.widgets['container'] + + # calls / puts header + type_header = BoxLayout( + orientation='horizontal', + size_hint=(1, 1/30.), + ) + calls = Label(text='calls', font_size='20') + puts = Label(text='puts', font_size='20') + type_header.add_widget(calls) + type_header.add_widget(puts) + container.add_widget(type_header) + + # figure out header fields for each table based on quote keys + headers = displayables[0].keys() + header_row = StrikeRow(strike='strike', size_hint=(1, None)) + header_record = {key: key for key in headers} + header_record['contract_type'] = 'put' + header_row.append_sub_row( + header_record, + header_record, + headers=headers, + bidasks=self.bidasks, + is_header=True, + size_hint=(1, None), + ) + header_record['contract_type'] = 'call' + header_row.append_sub_row( + header_record, + header_record, + headers=headers, + bidasks=self.bidasks, + is_header=True, + size_hint=(1, None), + ) + container.add_widget(header_row) + + # build out chain tables + table = TickerTable( + sort_key='strike', + cols=1, + size_hint=(1, None), + ) + header_row.table = table + table.bind(minimum_height=table.setter('height')) + pager = PagerView( + container=container, + contained=table, + nursery=self._nursery + ) + container.add_widget(pager) + + self.widgets.update({ + 'table': table, + 'type_header': type_header, + 'table': table, + 'pager': pager, + }) async def new_chain_ui( @@ -364,33 +479,6 @@ async def new_chain_ui( ) -> None: """Create and return a new option chain UI. """ - # retreive all contracts just because we need a default when the - # UI starts up - all_contracts = await contracts(brokermod, symbol) - # start streaming soonest contract by default - expiry = next(iter(all_contracts)).expiry - - widgets = {} - - # define bid-ask "stacked" cells - # (TODO: needs some rethinking and renaming for sure) - bidasks = brokermod._option_bidasks - - feed = DataFeed(portal, brokermod) - chain = OptionChain( - symbol, - expiry, - widgets, - bidasks, - feed, - rate=rate, - ) - quote_gen, first_quotes = await chain.feed.open_stream([chain.sub]) - records, displayables = zip(*[ - brokermod.format_option_quote(quote, {}) - for quote in first_quotes.values() - ]) - # use `monitor` styling for now from .monitor import _kv Builder.load_string(_kv) @@ -398,78 +486,30 @@ async def new_chain_ui( # the master container container = BoxLayout(orientation='vertical', spacing=0) - # TODO: figure out how to compact these buttons - expiries = { - key.expiry: key.expiry[:key.expiry.find('T')] - for key in all_contracts - } - expiry_buttons = Row( - record=expiries, - headers=expiries, - is_header=True, - size_hint=(1, None), - cell_type=ExpiryButton, - ) - # top row of expiry buttons - container.add_widget(expiry_buttons) - - # denote calls vs. puts side of table - type_header = BoxLayout( + # expiry buttons row (populated later once contracts are retreived) + expiry_row = BoxLayout( orientation='horizontal', - size_hint=(1, 1/28.), + size_hint=(1, None), ) - calls = Label(text='calls', font_size='20') - puts = Label(text='puts', font_size='20') - type_header.add_widget(calls) - type_header.add_widget(puts) - container.add_widget(type_header) + container.add_widget(expiry_row) - # figure out header fields for each table based on quote keys - headers = displayables[0].keys() - header_row = StrikeRow(strike='strike', size_hint=(1, None)) - header_record = {key: key for key in headers} - header_record['contract_type'] = 'put' - header_row.append_sub_row( - header_record, - header_record, - headers=headers, - bidasks=bidasks, - is_header=True, - size_hint=(1, None), - ) - header_record['contract_type'] = 'call' - header_row.append_sub_row( - header_record, - header_record, - headers=headers, - bidasks=bidasks, - is_header=True, - size_hint=(1, None), - ) - container.add_widget(header_row) - - # build out chain tables - table = TickerTable( - sort_key='strike', - cols=1, - size_hint=(1, None), - ) - header_row.table = table - table.bind(minimum_height=table.setter('height')) - pager = PagerView( - container=container, - contained=table, - nursery=nursery - ) - container.add_widget(pager) - widgets.update({ + widgets = { 'window': Window, 'root': container, 'container': container, - 'table': table, - 'expiry_buttons': expiry_buttons, - 'pager': pager, - }) + 'expiry_row': expiry_row, + } + # define bid-ask "stacked" cells + # (TODO: needs some rethinking and renaming for sure) + bidasks = brokermod._option_bidasks + + feed = DataFeed(portal, brokermod) + chain = OptionChain( + widgets, + bidasks, + feed, + rate=rate, + ) return chain @@ -493,9 +533,9 @@ async def _async_main( nursery, rate=rate, ) - async with chain.open_update_scope(): + async with chain.open_rt_display(nursery, symbol): try: - # Trio-kivy entry point. + # trio-kivy entry point. await async_runTouchApp(chain.widgets['root']) # run kivy finally: # cancel GUI update task From 32a7f4cbd381aeaa41ad82c4b4aabe9f5b6d898f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 31 Dec 2018 11:51:04 -0500 Subject: [PATCH 38/42] Right, gotta expose widgets to other actors --- piker/ui/monitor.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/piker/ui/monitor.py b/piker/ui/monitor.py index 1b6d8f3a..06a0c160 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/monitor.py @@ -706,6 +706,7 @@ async def _async_main( # set up a pager view for large ticker lists table.bind(minimum_height=table.setter('height')) + ss = tractor.current_actor().statespace try: async with trio.open_nursery() as nursery: pager = PagerView( @@ -722,6 +723,7 @@ async def _async_main( 'header': header, 'pager': pager, } + ss['widgets'] = widgets nursery.start_soon( update_quotes, nursery, From 0cffa4b97a66a3b52a9c76ffadecdb433d089276 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 1 Jan 2019 23:36:46 -0500 Subject: [PATCH 39/42] Font size shrinks --- piker/ui/monitor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/piker/ui/monitor.py b/piker/ui/monitor.py index 06a0c160..5518bb22 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/monitor.py @@ -49,6 +49,7 @@ def colorcode(name): _bs = 0.75 # border size +_fs = 20 # font size # medium shade of gray that seems to match the # default i3 window borders @@ -63,7 +64,7 @@ _kv = (f''' #:kivy 1.10.0 - font_size: 21 + font_size: {_fs} # make text wrap to botom text_size: self.size @@ -87,7 +88,7 @@ _kv = (f''' - font_size: 21 + font_size: {_fs} # canvas.before: # Color: # rgba: [0.13]*4 @@ -102,7 +103,7 @@ _kv = (f''' spacing: [{_bs}] # row_force_default: True - row_default_height: 62 + row_default_height: 56 cols: 1 canvas.before: Color: From 48956906429ec76ecf66955f538f4990fdbe76de Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 1 Jan 2019 23:42:49 -0500 Subject: [PATCH 40/42] Display a message when no contracts exist --- piker/ui/option_chain.py | 200 +++++++++++++++++++++------------------ 1 file changed, 109 insertions(+), 91 deletions(-) diff --git a/piker/ui/option_chain.py b/piker/ui/option_chain.py index 86654175..531ea0d5 100644 --- a/piker/ui/option_chain.py +++ b/piker/ui/option_chain.py @@ -37,6 +37,7 @@ class StrikeCell(HeaderCell): _no_display = ['symbol', 'contract_type', 'strike', 'time', 'open'] _strike_row_cache = {} _strike_cell_cache = {} +_no_contracts_msg = "No contracts available for symbol" class StrikeRow(BoxLayout): @@ -237,14 +238,22 @@ class OptionChain(object): self._nursery = None self._update_nursery = None self.feed = feed - self._update_cs = None self._quote_gen = None # TODO: this should be moved down to the data feed layer - # right now it's only needed for the UI uupdate loop to cancel itself + # right now it's only needed for the UI update loop to cancel itself + self._update_cs = None self._first_quotes = None self._last_expiry = None # flag to determine if one-time widgets have been generated self._static_widgets_initialized = False + self._no_opts_label = None + + @property + def no_opts_label(self): + if self._no_opts_label is None: + label = self._no_opts_label = Label(text=_no_contracts_msg) + label.font_size = 30 + return self._no_opts_label async def _rx_symbols(self): async with find_local_monitor() as portal: @@ -319,95 +328,6 @@ class OptionChain(object): log.debug("Finished rendering rows!") - async def _start_displaying(self, symbol, expiry=None): - """Main routine to start displaying the real time updated strike - table. - - Clear any existing data feed subscription that is no longer needed - (eg. when clicking a new expiry button) spin up a new subscription, - populate the table and start updating it. - """ - # redraw any symbol specific UI components - if self.symbol != symbol or expiry is None: - # set window title - self.widgets['window'].set_title( - self._title.format(symbol=symbol) - ) - - # retreive all contracts to populate expiry row - all_contracts = await contracts(self.feed.brokermod, symbol) - # start streaming soonest contract by default if not provided - expiry = next(iter(all_contracts)).expiry if not expiry else expiry - - # TODO: figure out how to compact these buttons - expiries = { - key.expiry: key.expiry[:key.expiry.find('T')] - for key in all_contracts - } - expiry_row = self.widgets['expiry_row'] - expiry_row.clear_widgets() - - for expiry, justdate in expiries.items(): - button = ExpiryButton(text=str(justdate), key=expiry) - # assign us to each expiry button - button.chain = self - expiry_row.add_widget(button) - - if self.widgets.get('table'): - self.clear_strikes() - - if self._update_cs: - log.warn("Cancelling existing update task") - self._update_cs.cancel() - await trio.sleep(0) - - if self._quote_gen: - await self._quote_gen.aclose() - - if self._nursery is None: - raise RuntimeError( - "You must call open this chain's update scope first!") - - log.debug(f"Waiting on first_quotes for {symbol}:{expiry}") - self._quote_gen, first_quotes = await self.feed.open_stream( - [(symbol, expiry)] - ) - log.debug(f"Got first_quotes for {symbol}:{expiry}") - records, displayables = self.feed.format_quotes(first_quotes) - - # draw static widgets only once - if self._static_widgets_initialized is False: - self._init_static_widgets(displayables) - self._static_widgets_initialized = True - - n = self._nursery - self.render_rows(records, displayables) - - with trio.open_cancel_scope() as cs: - self._update_cs = cs - await n.start( - partial( - update_quotes, - n, - self.feed.brokermod.format_option_quote, - self.widgets, - self._quote_gen, - symbol_data={}, - first_quotes=first_quotes, - ) - ) - self.symbol, self.expiry = symbol, expiry - - def start_displaying(self, symbol, expiry): - if self.symbol == symbol and self.expiry == expiry: - log.info(f"Clicked {symbol}:{expiry} is already selected") - return - - log.info(f"Subscribing for {symbol}:{expiry}") - self._nursery.start_soon( - partial(self._start_displaying, symbol, expiry=expiry) - ) - def _init_static_widgets(self, displayables): assert self._static_widgets_initialized is False container = self.widgets['container'] @@ -469,6 +389,104 @@ class OptionChain(object): 'pager': pager, }) + async def _start_displaying(self, symbol, expiry=None): + """Main routine to start displaying the real time updated strike + table. + + Clear any existing data feed subscription that is no longer needed + (eg. when clicking a new expiry button) spin up a new subscription, + populate the table and start updating it. + """ + table = self.widgets.get('table') + if table: + self.clear_strikes() + + if self._update_cs: + log.warn("Cancelling existing update task") + self._update_cs.cancel() + await trio.sleep(0) + + if self._quote_gen: + await self._quote_gen.aclose() + + # redraw any symbol specific UI components + if self.symbol != symbol or expiry is None: + # set window title + self.widgets['window'].set_title( + self._title.format(symbol=symbol) + ) + + # retreive all contracts to populate expiry row + all_contracts = await contracts(self.feed.brokermod, symbol) + + if not all_contracts: + label = self.no_opts_label + label.symbol = symbol + if table: + table.add_widget(label) + return + + # start streaming soonest contract by default if not provided + expiry = next(iter(all_contracts)).expiry if not expiry else expiry + + # TODO: figure out how to compact these buttons + expiries = { + key.expiry: key.expiry[:key.expiry.find('T')] + for key in all_contracts + } + expiry_row = self.widgets['expiry_row'] + expiry_row.clear_widgets() + + for expiry, justdate in expiries.items(): + button = ExpiryButton(text=str(justdate), key=expiry) + # assign us to each expiry button + button.chain = self + expiry_row.add_widget(button) + + if self._nursery is None: + raise RuntimeError( + "You must call open this chain's update scope first!") + + log.debug(f"Waiting on first_quotes for {symbol}:{expiry}") + self._quote_gen, first_quotes = await self.feed.open_stream( + [(symbol, expiry)] + ) + log.debug(f"Got first_quotes for {symbol}:{expiry}") + records, displayables = self.feed.format_quotes(first_quotes) + + # draw static widgets only once + if self._static_widgets_initialized is False: + self._init_static_widgets(displayables) + self._static_widgets_initialized = True + + self.render_rows(records, displayables) + + with trio.open_cancel_scope() as cs: + self._update_cs = cs + await self._nursery.start( + partial( + update_quotes, + self._nursery, + self.feed.brokermod.format_option_quote, + self.widgets, + self._quote_gen, + symbol_data={}, + first_quotes=first_quotes, + ) + ) + # always keep track of current subscription + self.symbol, self.expiry = symbol, expiry + + def start_displaying(self, symbol, expiry): + if self.symbol == symbol and self.expiry == expiry: + log.info(f"Clicked {symbol}:{expiry} is already selected") + return + + log.info(f"Subscribing for {symbol}:{expiry}") + self._nursery.start_soon( + partial(self._start_displaying, symbol, expiry=expiry) + ) + async def new_chain_ui( portal: tractor._portal.Portal, From fa6bae1f5cd348d781c884c06340d68b498a30e6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 2 Jan 2019 21:12:42 -0500 Subject: [PATCH 41/42] Reorg table widgets into a new module --- piker/ui/monitor.py | 463 +------------------------------------- piker/ui/option_chain.py | 3 +- piker/ui/tabular.py | 470 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 475 insertions(+), 461 deletions(-) create mode 100644 piker/ui/tabular.py diff --git a/piker/ui/monitor.py b/piker/ui/monitor.py index 5518bb22..537227ef 100644 --- a/piker/ui/monitor.py +++ b/piker/ui/monitor.py @@ -5,483 +5,26 @@ Launch with ``piker monitor ``. (Currently there's a bunch of questrade specific stuff in here) """ -from itertools import chain from types import ModuleType, AsyncGeneratorType from typing import List, Callable -from bisect import bisect import trio import tractor from kivy.uix.boxlayout import BoxLayout -from kivy.uix.gridlayout import GridLayout -from kivy.uix.stacklayout import StackLayout -from kivy.uix.button import Button from kivy.lang import Builder -from kivy import utils from kivy.app import async_runTouchApp from kivy.core.window import Window -from kivy.properties import BooleanProperty +from .tabular import ( + Row, TickerTable, _kv, _black_rgba, colorcode, +) from ..log import get_logger from .pager import PagerView -from .kivy.mouse_over import new_mouse_over_group -HoverBehavior = new_mouse_over_group() log = get_logger('monitor') -_colors2hexs = { - 'darkgray': 'a9a9a9', - 'gray': '808080', - 'green': '008000', - 'forestgreen': '228b22', - 'red2': 'ff3333', - 'red': 'ff0000', - 'firebrick': 'b22222', -} - -_colors = {key: utils.rgba(val) for key, val in _colors2hexs.items()} - - -def colorcode(name): - return _colors[name if name else 'gray'] - - -_bs = 0.75 # border size -_fs = 20 # font size - -# medium shade of gray that seems to match the -# default i3 window borders -_i3_rgba = [0.14]*3 + [1] - -# slightly off black like the jellybean bg from -# vim colorscheme -_cell_rgba = [0.07]*3 + [1] -_black_rgba = [0]*4 - -_kv = (f''' -#:kivy 1.10.0 - - - font_size: {_fs} - - # make text wrap to botom - text_size: self.size - halign: 'center' - valign: 'middle' - size: self.texture_size - - # don't set these as the update loop already does it - # color: {colorcode('gray')} - # font_color: {colorcode('gray')} - # font_name: 'Hack-Regular' - - # if `highlight` is set use i3 color by default transparent; use row color - # this is currently used for expiry cells on the options chain - background_color: {_i3_rgba} if self.click_toggle else {_black_rgba} - # must be set to allow 'plain bg colors' since default texture is grey - # but right now is only set for option chain expiry buttons - # background_normal: '' - # spacing: 0, 0 - # padding: 3, 3 - - - - font_size: {_fs} - # canvas.before: - # Color: - # rgba: [0.13]*4 - # BorderImage: # use a fixed size border - # pos: self.pos - # size: [self.size[0] - {_bs}, self.size[1]] - # # 0s are because the containing TickerTable already has spacing - # # border: [0, {_bs} , 0, {_bs}] - # border: [0, {_bs} , 0, 0] - - - - spacing: [{_bs}] - # row_force_default: True - row_default_height: 56 - cols: 1 - canvas.before: - Color: - # i3 style gray as background - rgba: {_i3_rgba} - Rectangle: - # scale with container self here refers to the widget i.e BoxLayout - pos: self.pos - size: self.size - - - - spacing: [{_bs}, 0] - - - - # minimum_height: 200 # should be pulled from Cell text size - # minimum_width: 200 - # row_force_default: True - # row_default_height: 61 # determines the header row size - padding: [0]*4 - spacing: [0] - canvas.before: - Color: - rgba: {_cell_rgba} - Rectangle: - # self here refers to the widget i.e Row(GridLayout) - pos: self.pos - size: self.size - # row higlighting on mouse over - Color: - rgba: {_i3_rgba} - # RoundedRectangle: - Rectangle: - size: self.width, self.height if self.hovered else 1 - pos: self.pos - # radius: (0,) - - -# part of the `PagerView` - - size_hint: 1, None - # static size of 51 px - height: 51 - font_size: 25 - background_color: {_i3_rgba} -''') - - -class Cell(Button): - """Data cell: the fundemental widget. - - ``key`` is the column name index value. - """ - click_toggle = BooleanProperty(False) - - def __init__(self, key=None, is_header=False, **kwargs): - super(Cell, self).__init__(**kwargs) - self.key = key - self.row = None - self.is_header = is_header - - def on_press(self, value=None): - self.row.on_press() - - -class HeaderCell(Cell): - """Column header cell label. - """ - def on_press(self, value=None): - """Clicking on a col header indicates to sort rows by this column - in `update_quotes()`. - """ - table = self.row.table - # if this is a row header cell then sort by the clicked field - if self.row.is_header: - table.sort_key = self.key - - last = table.last_clicked_col_cell - if last and last is not self: - last.underline = False - last.bold = False - - # outline the header text to indicate it's been the last clicked - self.underline = True - self.bold = True - # mark this cell as the last selected - table.last_clicked_col_cell = self - # sort and render the rows immediately - self.row.table.render_rows(table.symbols2rows.values()) - - # TODO: make this some kind of small geometry instead - # (maybe like how trading view does it). - # allow highlighting of row headers for tracking - elif self.is_header: - if self.background_color == self.color: - self.background_color = _black_rgba - else: - self.background_color = self.color - - -class BidAskLayout(StackLayout): - """Cell which houses three buttons containing a last, bid, and ask in a - single unit oriented with the last 2 under the first. - """ - def __init__(self, values, header=False, **kwargs): - # uncomment to get vertical stacked bid-ask - # super(BidAskLayout, self).__init__(orientation='bt-lr', **kwargs) - super(BidAskLayout, self).__init__(orientation='lr-tb', **kwargs) - assert len(values) == 3, "You can only provide 3 values: last,bid,ask" - self._keys2cells = {} - cell_type = HeaderCell if header else Cell - top_size = cell_type().font_size - small_size = top_size - 4 - top_prop = 0.5 # proportion of size used by top cell - bottom_prop = 1 - top_prop - for (key, size_hint, font_size), value in zip( - [('last', (1, top_prop), top_size), - ('bid', (0.5, bottom_prop), small_size), - ('ask', (0.5, bottom_prop), small_size)], - # uncomment to get vertical stacked bid-ask - # [('last', (top_prop, 1), top_size), - # ('bid', (bottom_prop, 0.5), small_size), - # ('ask', (bottom_prop, 0.5), small_size)], - values - ): - cell = cell_type( - text=str(value), - size_hint=size_hint, - # width=self.width/2 - 3, - font_size=font_size - ) - self._keys2cells[key] = cell - cell.key = value - cell.is_header = header - setattr(self, key, cell) - self.add_widget(cell) - - # should be assigned by referrer - self.row = None - - def get_cell(self, key): - return self._keys2cells.get(key) - - @property - def row(self): - return self.row - - @row.setter - def row(self, row): - # so hideous - for cell in self.cells: - cell.row = row - - @property - def cells(self): - return [self.last, self.bid, self.ask] - - -class Row(HoverBehavior, GridLayout): - """A grid for displaying a row of ticker quote data. - """ - def __init__( - self, - record, - headers=(), - no_cell=(), - bidasks=None, - table=None, - is_header=False, - cell_type=None, - **kwargs - ): - super().__init__(cols=len(record), **kwargs) - self._cell_widgets = {} - self._last_record = record - self.table = table - self.is_header = is_header - self._cell_type = cell_type - self.widget = self - - # Create `BidAskCells` first. - # bid/ask cells are just 3 cells grouped in a - # ``BidAskLayout`` which just stacks the parent cell - # on top of 2 children. - layouts = {} - bidasks = bidasks or {} - ba_cells = {} - for key, children in bidasks.items(): - layout = BidAskLayout( - [record[key]] + [record[child] for child in children], - header=is_header - ) - layout.row = self - layouts[key] = layout - for i, child in enumerate([key] + children): - ba_cells[child] = layout.cells[i] - - children_flat = list(chain.from_iterable(bidasks.values())) - self._cell_widgets.update(ba_cells) - - # build out row using Cell labels - for (key, val) in record.items(): - header = key in headers - - # handle bidask cells - if key in layouts: - self.add_widget(layouts[key]) - elif key in children_flat: - # these cells have already been added to the `BidAskLayout` - continue - elif key not in no_cell: - cell = self._append_cell(val, key, header=header) - cell.key = key - self._cell_widgets[key] = cell - - def iter_cells(self): - return self._cell_widgets.items() - - def get_cell(self, key): - return self._cell_widgets.get(key) - - def get_field(self, key): - return self._last_record[key] - - def _append_cell(self, text, key, header=False): - if not len(self._cell_widgets) < self.cols: - raise ValueError(f"Can not append more then {self.cols} cells") - - # header cells just have a different colour - celltype = self._cell_type or (HeaderCell if header else Cell) - cell = celltype(text=str(text), key=key) - cell.is_header = header - cell.row = self - self.add_widget(cell) - return cell - - def update(self, record, displayable): - """Update this row's cells with new values from a quote - ``record``. - - Return all cells that changed in a ``dict``. - """ - # color changed field values - cells = {} - gray = colorcode('gray') - fgreen = colorcode('forestgreen') - red = colorcode('red2') - for key, val in record.items(): - last = self.get_field(key) - color = gray - try: - # logic for cell text coloring: up-green, down-red - if last < val: - color = fgreen - elif last > val: - color = red - except TypeError: - log.warn(f"wtf QT {val} is not regular?") - - cell = self.get_cell(key) - # some displayable fields might have specifically - # not had cells created as set in the `no_cell` attr - if cell is not None: - cell.text = str(displayable[key]) - cell.color = color - if color != gray: - cells[key] = cell - - self._last_record = record - return cells - - # mouse over handlers - def on_enter(self): - """Highlight layout on enter. - """ - log.debug( - f"Entered row {self} through {self.border_point}") - # don't highlight header row - if self.is_header: - self.hovered = False - - def on_leave(self): - """Un-highlight layout on exit. - """ - log.debug( - f"Left row {self} through {self.border_point}") - - def on_press(self, value=None): - log.info(f"Pressed row for {self._last_record['symbol']}") - if self.table and not self.is_header: - for q in self.table._click_queues: - q.put_nowait(self._last_record['symbol']) - - -class TickerTable(GridLayout): - """A grid for displaying ticker quote records as a table. - """ - def __init__(self, sort_key='%', auto_sort=True, **kwargs): - super(TickerTable, self).__init__(**kwargs) - self.symbols2rows = {} - self.sort_key = sort_key - # for tracking last clicked column header cell - self.last_clicked_col_cell = None - self._auto_sort = auto_sort - self._symbols2index = {} - self._sorted = [] - self._click_queues: List[trio.Queue] = [] - - def append_row(self, key, row): - """Append a `Row` of `Cell` objects to this table. - """ - # store ref to each row - self.symbols2rows[key] = row - self.add_widget(row) - self._sorted.append(row) - return row - - def clear(self): - self.clear_widgets() - self._sorted.clear() - - def render_rows( - self, - changed: set, - sort_key: str = None, - ): - """Sort and render all rows on the ticker grid from ``syms2rows``. - """ - sort_key = sort_key or self.sort_key - key_row_pairs = list(sorted( - [(row.get_field(sort_key), row) for row in self._sorted], - key=lambda item: item[0], - )) - if key_row_pairs: - sorted_keys, sorted_rows = zip(*key_row_pairs) - sorted_keys, sorted_rows = list(sorted_keys), list(sorted_rows) - else: - sorted_keys, sorted_rows = [], [] - - # now remove and re-insert any rows that need to be shuffled - # due to new a new field change - for row in changed: - try: - old_index = sorted_rows.index(row) - except ValueError: - # row is not yet added so nothing to remove - pass - else: - del sorted_rows[old_index] - del sorted_keys[old_index] - self._sorted.remove(row) - self.remove_widget(row) - - for row in changed: - key = row.get_field(sort_key) - index = bisect(sorted_keys, key) - sorted_keys.insert(index, key) - self._sorted.insert(index, row) - self.add_widget(row, index=index) - - def ticker_search(self, patt): - """Return sequence of matches when pattern ``patt`` is in a - symbol name. Most naive algo possible for the moment. - """ - for symbol, row in self.symbols2rows.items(): - if patt in symbol: - yield symbol, row - - def get_row(self, symbol: str) -> Row: - return self.symbols2rows[symbol] - - def search(self, patt): - """Search bar api compat. - """ - return dict(self.ticker_search(patt)) or {} - - async def update_quotes( nursery: trio._core._run.Nursery, formatter: Callable, diff --git a/piker/ui/option_chain.py b/piker/ui/option_chain.py index 531ea0d5..237c8431 100644 --- a/piker/ui/option_chain.py +++ b/piker/ui/option_chain.py @@ -20,7 +20,8 @@ from ..log import get_logger from ..brokers.core import contracts from .pager import PagerView -from .monitor import Row, HeaderCell, Cell, TickerTable, update_quotes +from .tabular import Row, HeaderCell, Cell, TickerTable +from .monitor import update_quotes log = get_logger('option_chain') diff --git a/piker/ui/tabular.py b/piker/ui/tabular.py new file mode 100644 index 00000000..7a9aca2f --- /dev/null +++ b/piker/ui/tabular.py @@ -0,0 +1,470 @@ +""" +Real-time table components +""" +from itertools import chain +from typing import List +from bisect import bisect + +import trio +from kivy.uix.gridlayout import GridLayout +from kivy.uix.stacklayout import StackLayout +from kivy.uix.button import Button +from kivy import utils +from kivy.properties import BooleanProperty + +from ..log import get_logger +from .kivy.mouse_over import new_mouse_over_group + + +HoverBehavior = new_mouse_over_group() +log = get_logger('monitor') + +_colors2hexs = { + 'darkgray': 'a9a9a9', + 'gray': '808080', + 'green': '008000', + 'forestgreen': '228b22', + 'red2': 'ff3333', + 'red': 'ff0000', + 'firebrick': 'b22222', +} + +_colors = {key: utils.rgba(val) for key, val in _colors2hexs.items()} + + +def colorcode(name): + return _colors[name if name else 'gray'] + + +_bs = 0.75 # border size +_fs = 20 # font size + +# medium shade of gray that seems to match the +# default i3 window borders +_i3_rgba = [0.14]*3 + [1] + +# slightly off black like the jellybean bg from +# vim colorscheme +_cell_rgba = [0.07]*3 + [1] +_black_rgba = [0]*4 + +_kv = (f''' +#:kivy 1.10.0 + + + font_size: {_fs} + + # make text wrap to botom + text_size: self.size + halign: 'center' + valign: 'middle' + size: self.texture_size + + # don't set these as the update loop already does it + # color: {colorcode('gray')} + # font_color: {colorcode('gray')} + # font_name: 'Hack-Regular' + + # if `highlight` is set use i3 color by default transparent; use row color + # this is currently used for expiry cells on the options chain + background_color: {_i3_rgba} if self.click_toggle else {_black_rgba} + # must be set to allow 'plain bg colors' since default texture is grey + # but right now is only set for option chain expiry buttons + # background_normal: '' + # spacing: 0, 0 + # padding: 3, 3 + + + + font_size: {_fs} + # canvas.before: + # Color: + # rgba: [0.13]*4 + # BorderImage: # use a fixed size border + # pos: self.pos + # size: [self.size[0] - {_bs}, self.size[1]] + # # 0s are because the containing TickerTable already has spacing + # # border: [0, {_bs} , 0, {_bs}] + # border: [0, {_bs} , 0, 0] + + + + spacing: [{_bs}] + # row_force_default: True + row_default_height: 56 + cols: 1 + canvas.before: + Color: + # i3 style gray as background + rgba: {_i3_rgba} + Rectangle: + # scale with container self here refers to the widget i.e BoxLayout + pos: self.pos + size: self.size + + + + spacing: [{_bs}, 0] + + + + # minimum_height: 200 # should be pulled from Cell text size + # minimum_width: 200 + # row_force_default: True + # row_default_height: 61 # determines the header row size + padding: [0]*4 + spacing: [0] + canvas.before: + Color: + rgba: {_cell_rgba} + Rectangle: + # self here refers to the widget i.e Row(GridLayout) + pos: self.pos + size: self.size + # row higlighting on mouse over + Color: + rgba: {_i3_rgba} + # RoundedRectangle: + Rectangle: + size: self.width, self.height if self.hovered else 1 + pos: self.pos + # radius: (0,) + + +# part of the `PagerView` + + size_hint: 1, None + # static size of 51 px + height: 51 + font_size: 25 + background_color: {_i3_rgba} +''') + + +class Cell(Button): + """Data cell: the fundemental widget. + + ``key`` is the column name index value. + """ + click_toggle = BooleanProperty(False) + + def __init__(self, key=None, is_header=False, **kwargs): + super(Cell, self).__init__(**kwargs) + self.key = key + self.row = None + self.is_header = is_header + + def on_press(self, value=None): + self.row.on_press() + + +class HeaderCell(Cell): + """Column header cell label. + """ + def on_press(self, value=None): + """Clicking on a col header indicates to sort rows by this column + in `update_quotes()`. + """ + table = self.row.table + # if this is a row header cell then sort by the clicked field + if self.row.is_header: + table.sort_key = self.key + + last = table.last_clicked_col_cell + if last and last is not self: + last.underline = False + last.bold = False + + # outline the header text to indicate it's been the last clicked + self.underline = True + self.bold = True + # mark this cell as the last selected + table.last_clicked_col_cell = self + # sort and render the rows immediately + self.row.table.render_rows(table.symbols2rows.values()) + + # TODO: make this some kind of small geometry instead + # (maybe like how trading view does it). + # allow highlighting of row headers for tracking + elif self.is_header: + if self.background_color == self.color: + self.background_color = _black_rgba + else: + self.background_color = self.color + + +class BidAskLayout(StackLayout): + """Cell which houses three buttons containing a last, bid, and ask in a + single unit oriented with the last 2 under the first. + """ + def __init__(self, values, header=False, **kwargs): + # uncomment to get vertical stacked bid-ask + # super(BidAskLayout, self).__init__(orientation='bt-lr', **kwargs) + super(BidAskLayout, self).__init__(orientation='lr-tb', **kwargs) + assert len(values) == 3, "You can only provide 3 values: last,bid,ask" + self._keys2cells = {} + cell_type = HeaderCell if header else Cell + top_size = cell_type().font_size + small_size = top_size - 4 + top_prop = 0.5 # proportion of size used by top cell + bottom_prop = 1 - top_prop + for (key, size_hint, font_size), value in zip( + [('last', (1, top_prop), top_size), + ('bid', (0.5, bottom_prop), small_size), + ('ask', (0.5, bottom_prop), small_size)], + # uncomment to get vertical stacked bid-ask + # [('last', (top_prop, 1), top_size), + # ('bid', (bottom_prop, 0.5), small_size), + # ('ask', (bottom_prop, 0.5), small_size)], + values + ): + cell = cell_type( + text=str(value), + size_hint=size_hint, + # width=self.width/2 - 3, + font_size=font_size + ) + self._keys2cells[key] = cell + cell.key = value + cell.is_header = header + setattr(self, key, cell) + self.add_widget(cell) + + # should be assigned by referrer + self.row = None + + def get_cell(self, key): + return self._keys2cells.get(key) + + @property + def row(self): + return self.row + + @row.setter + def row(self, row): + # so hideous + for cell in self.cells: + cell.row = row + + @property + def cells(self): + return [self.last, self.bid, self.ask] + + +class Row(HoverBehavior, GridLayout): + """A grid for displaying a row of ticker quote data. + """ + def __init__( + self, + record, + headers=(), + no_cell=(), + bidasks=None, + table=None, + is_header=False, + cell_type=None, + **kwargs + ): + super().__init__(cols=len(record), **kwargs) + self._cell_widgets = {} + self._last_record = record + self.table = table + self.is_header = is_header + self._cell_type = cell_type + self.widget = self + + # Create `BidAskCells` first. + # bid/ask cells are just 3 cells grouped in a + # ``BidAskLayout`` which just stacks the parent cell + # on top of 2 children. + layouts = {} + bidasks = bidasks or {} + ba_cells = {} + for key, children in bidasks.items(): + layout = BidAskLayout( + [record[key]] + [record[child] for child in children], + header=is_header + ) + layout.row = self + layouts[key] = layout + for i, child in enumerate([key] + children): + ba_cells[child] = layout.cells[i] + + children_flat = list(chain.from_iterable(bidasks.values())) + self._cell_widgets.update(ba_cells) + + # build out row using Cell labels + for (key, val) in record.items(): + header = key in headers + + # handle bidask cells + if key in layouts: + self.add_widget(layouts[key]) + elif key in children_flat: + # these cells have already been added to the `BidAskLayout` + continue + elif key not in no_cell: + cell = self._append_cell(val, key, header=header) + cell.key = key + self._cell_widgets[key] = cell + + def iter_cells(self): + return self._cell_widgets.items() + + def get_cell(self, key): + return self._cell_widgets.get(key) + + def get_field(self, key): + return self._last_record[key] + + def _append_cell(self, text, key, header=False): + if not len(self._cell_widgets) < self.cols: + raise ValueError(f"Can not append more then {self.cols} cells") + + # header cells just have a different colour + celltype = self._cell_type or (HeaderCell if header else Cell) + cell = celltype(text=str(text), key=key) + cell.is_header = header + cell.row = self + self.add_widget(cell) + return cell + + def update(self, record, displayable): + """Update this row's cells with new values from a quote + ``record``. + + Return all cells that changed in a ``dict``. + """ + # color changed field values + cells = {} + gray = colorcode('gray') + fgreen = colorcode('forestgreen') + red = colorcode('red2') + for key, val in record.items(): + last = self.get_field(key) + color = gray + try: + # logic for cell text coloring: up-green, down-red + if last < val: + color = fgreen + elif last > val: + color = red + except TypeError: + log.warn(f"wtf QT {val} is not regular?") + + cell = self.get_cell(key) + # some displayable fields might have specifically + # not had cells created as set in the `no_cell` attr + if cell is not None: + cell.text = str(displayable[key]) + cell.color = color + if color != gray: + cells[key] = cell + + self._last_record = record + return cells + + # mouse over handlers + def on_enter(self): + """Highlight layout on enter. + """ + log.debug( + f"Entered row {self} through {self.border_point}") + # don't highlight header row + if self.is_header: + self.hovered = False + + def on_leave(self): + """Un-highlight layout on exit. + """ + log.debug( + f"Left row {self} through {self.border_point}") + + def on_press(self, value=None): + log.info(f"Pressed row for {self._last_record['symbol']}") + if self.table and not self.is_header: + for q in self.table._click_queues: + q.put_nowait(self._last_record['symbol']) + + +class TickerTable(GridLayout): + """A grid for displaying ticker quote records as a table. + """ + def __init__(self, sort_key='%', auto_sort=True, **kwargs): + super(TickerTable, self).__init__(**kwargs) + self.symbols2rows = {} + self.sort_key = sort_key + # for tracking last clicked column header cell + self.last_clicked_col_cell = None + self._auto_sort = auto_sort + self._symbols2index = {} + self._sorted = [] + self._click_queues: List[trio.Queue] = [] + + def append_row(self, key, row): + """Append a `Row` of `Cell` objects to this table. + """ + # store ref to each row + self.symbols2rows[key] = row + self.add_widget(row) + self._sorted.append(row) + return row + + def clear(self): + self.clear_widgets() + self._sorted.clear() + + def render_rows( + self, + changed: set, + sort_key: str = None, + ): + """Sort and render all rows on the ticker grid from ``syms2rows``. + """ + sort_key = sort_key or self.sort_key + key_row_pairs = list(sorted( + [(row.get_field(sort_key), row) for row in self._sorted], + key=lambda item: item[0], + )) + if key_row_pairs: + sorted_keys, sorted_rows = zip(*key_row_pairs) + sorted_keys, sorted_rows = list(sorted_keys), list(sorted_rows) + else: + sorted_keys, sorted_rows = [], [] + + # now remove and re-insert any rows that need to be shuffled + # due to new a new field change + for row in changed: + try: + old_index = sorted_rows.index(row) + except ValueError: + # row is not yet added so nothing to remove + pass + else: + del sorted_rows[old_index] + del sorted_keys[old_index] + self._sorted.remove(row) + self.remove_widget(row) + + for row in changed: + key = row.get_field(sort_key) + index = bisect(sorted_keys, key) + sorted_keys.insert(index, key) + self._sorted.insert(index, row) + self.add_widget(row, index=index) + + def ticker_search(self, patt): + """Return sequence of matches when pattern ``patt`` is in a + symbol name. Most naive algo possible for the moment. + """ + for symbol, row in self.symbols2rows.items(): + if patt in symbol: + yield symbol, row + + def get_row(self, symbol: str) -> Row: + return self.symbols2rows[symbol] + + def search(self, patt): + """Search bar api compat. + """ + return dict(self.ticker_search(patt)) or {} From 7f8c88be0cb3e46b918665fec93a8a21df4fbedb Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 2 Jan 2019 21:24:42 -0500 Subject: [PATCH 42/42] Drop open/close prices for now; never really use them --- piker/brokers/questrade.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 2a588562..c3fb7586 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -583,11 +583,11 @@ _qt_stock_keys = { 'bidSize': 'bsize', 'askSize': 'asize', 'VWAP': ('VWAP', partial(round, ndigits=3)), - 'mktcap': ('mktcap', humanize), + 'MC': ('MC', humanize), '$ vol': ('$ vol', humanize), 'volume': ('vol', humanize), - 'close': 'close', - 'openPrice': 'open', + # 'close': 'close', + # 'openPrice': 'open', 'lowPrice': 'low', 'highPrice': 'high', # 'low52w': 'low52w', # put in info widget @@ -607,7 +607,7 @@ _stock_bidasks = { 'last': ['bid', 'ask'], 'size': ['bsize', 'asize'], 'VWAP': ['low', 'high'], - 'vol': ['mktcap', '$ vol'], + 'vol': ['MC', '$ vol'], } @@ -633,7 +633,7 @@ def format_stock_quote( computed = { 'symbol': quote['symbol'], '%': round(change, 3), - 'mktcap': mktcap, + 'MC': mktcap, # why QT do you have to be an asshole shipping null values!!! '$ vol': round((quote['VWAP'] or 0) * (quote['volume'] or 0), 3), 'close': previous,