From 27aed85404d9a13e5dcfe73f11ec30577227bf55 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 10 Apr 2021 12:20:21 -0400 Subject: [PATCH 01/81] Handle no matching symbols case --- piker/brokers/ib.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 17c0ceb2..112e935e 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -245,27 +245,31 @@ class Client: """ descriptions = await self.ib.reqMatchingSymbolsAsync(pattern) - futs = [] - for d in descriptions: - con = d.contract - futs.append(self.ib.reqContractDetailsAsync(con)) - - # batch request all details - results = await asyncio.gather(*futs) - - # XXX: if there is more then one entry in the details list - details = {} - for details_set in results: - # then the contract is so called "ambiguous". - for d in details_set: + if descriptions is not None: + futs = [] + for d in descriptions: con = d.contract - unique_sym = f'{con.symbol}.{con.primaryExchange}' - details[unique_sym] = asdict(d) if asdicts else d + futs.append(self.ib.reqContractDetailsAsync(con)) - if len(details) == upto: - return details + # batch request all details + results = await asyncio.gather(*futs) - return details + # XXX: if there is more then one entry in the details list + details = {} + for details_set in results: + # then the contract is so called "ambiguous". + for d in details_set: + con = d.contract + unique_sym = f'{con.symbol}.{con.primaryExchange}' + details[unique_sym] = asdict(d) if asdicts else d + + if len(details) == upto: + return details + + return details + + else: + return {} async def search_futes( self, @@ -562,7 +566,7 @@ class Client: # default config ports _tws_port: int = 7497 _gw_port: int = 4002 -_try_ports = [_tws_port, _gw_port] +_try_ports = [_gw_port, _tws_port] _client_ids = itertools.count() _client_cache = {} From 0627f7dceee77fee2238621ac703994152c9b8a8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 12 Apr 2021 18:04:26 -0400 Subject: [PATCH 02/81] First draft: symbol switching via QLineEdit widget --- piker/ui/_chart.py | 247 ++++++++++++++++++++++++++++----------- piker/ui/_interaction.py | 8 ++ 2 files changed, 189 insertions(+), 66 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 9f6e118b..f1ac6971 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -24,6 +24,8 @@ from types import ModuleType from functools import partial from PyQt5 import QtCore, QtGui +from PyQt5.QtCore import Qt +from PyQt5 import QtWidgets import numpy as np import pyqtgraph as pg import tractor @@ -48,6 +50,7 @@ from ._graphics._ohlc import BarItems from ._graphics._curve import FastAppendCurve from ._style import ( _font, + DpiAwareFont, hcolor, CHART_MARGINS, _xaxis_at, @@ -70,6 +73,99 @@ from .. import fsp log = get_logger(__name__) +class FontSizedQLineEdit(QtWidgets.QLineEdit): + + def __init__( + self, + parent_chart: 'ChartSpace', + font: DpiAwareFont = _font, + ) -> None: + super().__init__(parent_chart) + + self.dpi_font = font + self.chart_app = parent_chart + + # size it as we specify + self.setSizePolicy( + QtWidgets.QSizePolicy.Fixed, + QtWidgets.QSizePolicy.Fixed, + ) + self.setFont(font.font) + + # witty bit of margin + self.setTextMargins(2, 2, 2, 2) + + def sizeHint(self) -> QtCore.QSize: + psh = super().sizeHint() + psh.setHeight(self.dpi_font.px_size + 2) + return psh + + def unfocus(self) -> None: + self.hide() + self.clearFocus() + + def keyPressEvent(self, ev: QtCore.QEvent) -> None: + # by default we don't markt it as consumed? + # ev.ignore() + super().keyPressEvent(ev) + + ev.accept() + # text = ev.text() + key = ev.key() + mods = ev.modifiers() + + ctrl = False + if mods == QtCore.Qt.ControlModifier: + ctrl = True + + if ctrl: + if key == QtCore.Qt.Key_C: + self.unfocus() + self.chart_app.linkedcharts.focus() + + # TODO: + elif key == QtCore.Qt.Key_K: + print('move up') + + elif key == QtCore.Qt.Key_J: + print('move down') + + elif key in (QtCore.Qt.Key_Enter, QtCore.Qt.Key_Return): + print(f'Requesting symbol: {self.text()}') + symbol = self.text() + app = self.chart_app + self.chart_app.load_symbol( + app.linkedcharts.symbol.brokers[0], + symbol, + 'info', + ) + # self.hide() + self.unfocus() + + # if self._executing(): + # # ignore all key presses while executing, except for Ctrl-C + # if event.modifiers() == Qt.ControlModifier and key == Qt.Key_C: + # self._handle_ctrl_c() + # return True + + # handler = self._key_event_handlers.get(key) + # intercepted = handler and handler(event) + + # Assumes that Control+Key is a movement command, i.e. should not be + # handled as text insertion. However, on win10 AltGr is reported as + # Alt+Control which is why we handle this case like regular + # # keypresses, see #53: + # if not event.modifiers() & Qt.ControlModifier or \ + # event.modifiers() & Qt.AltModifier: + # self._keep_cursor_in_buffer() + + # if not intercepted and event.text(): + # intercepted = True + # self.insert_input_text(event.text()) + + # return False + + class ChartSpace(QtGui.QWidget): """High level widget which contains layouts for organizing lower level charts as well as other widgets used to control @@ -80,7 +176,7 @@ class ChartSpace(QtGui.QWidget): self.v_layout = QtGui.QVBoxLayout(self) self.v_layout.setContentsMargins(0, 0, 0, 0) - self.v_layout.setSpacing(0) + self.v_layout.setSpacing(1) self.toolbar_layout = QtGui.QHBoxLayout() self.toolbar_layout.setContentsMargins(0, 0, 0, 0) @@ -93,21 +189,18 @@ class ChartSpace(QtGui.QWidget): self.v_layout.addLayout(self.toolbar_layout) self.v_layout.addLayout(self.h_layout) self._chart_cache = {} + self.linkedcharts: 'LinkedSplitCharts' = None self.symbol_label: Optional[QtGui.QLabel] = None - def init_search(self): - self.symbol_label = label = QtGui.QLabel() - label.setTextFormat(3) # markdown - label.setFont(_font.font) - label.setMargin(0) - # title = f'sym: {self.symbol}' - # label.setText(title) + self._root_n: Optional[trio.Nursery] = None - label.setAlignment( - QtCore.Qt.AlignVCenter - | QtCore.Qt.AlignLeft - ) - self.v_layout.addWidget(label) + def open_search(self): + # search = self.search = QtWidgets.QLineEdit() + self.search = FontSizedQLineEdit(self) + self.search.unfocus() + self.v_layout.addWidget(self.search) + + # search.installEventFilter(self) def init_timeframes_ui(self): self.tf_layout = QtGui.QHBoxLayout() @@ -130,38 +223,59 @@ class ChartSpace(QtGui.QWidget): # def init_strategy_ui(self): # self.strategy_box = StrategyBoxWidget(self) # self.toolbar_layout.addWidget(self.strategy_box) + def load_symbol( self, brokername: str, symbol_key: str, - data: np.ndarray, + loglevel: str, ohlc: bool = True, + reset: bool = False, ) -> None: """Load a new contract into the charting app. Expects a ``numpy`` structured array containing all the ohlcv fields. + """ - # TODO: symbol search - # # of course this doesn't work :eyeroll: - # h = _font.boundingRect('Ag').height() - # print(f'HEIGHT {h}') - # self.symbol_label.setFixedHeight(h + 4) - # self.v_layout.update() - # self.symbol_label.setText(f'/`{symbol}`') + linkedcharts = self._chart_cache.get(symbol_key) - linkedcharts = self._chart_cache.setdefault( - symbol_key, - LinkedSplitCharts(self) - ) - self.linkedcharts = linkedcharts - - # remove any existing plots + # switching to a new viewable chart if not self.v_layout.isEmpty(): - self.v_layout.removeWidget(linkedcharts) + # and not ( + # self.linkedcharts is linkedcharts + # ): + # XXX: this is CRITICAL especially with pixel buffer caching + self.linkedcharts.hide() - self.v_layout.addWidget(linkedcharts) + # remove any existing plots + self.v_layout.removeWidget(self.linkedcharts) - return linkedcharts + if linkedcharts is None or reset: + + # we must load a fresh linked charts set + linkedcharts = LinkedSplitCharts(self) + self._root_n.start_soon( + chart_symbol, + self, + brokername, + symbol_key, + loglevel, + ) + self.v_layout.addWidget(linkedcharts) + + # if linkedcharts.chart: + # breakpoint() + + # else: + # chart is already in memory so just focus it + if self.linkedcharts: + self.linkedcharts.unfocus() + + # self.v_layout.addWidget(linkedcharts) + self.linkedcharts = linkedcharts + linkedcharts.focus() + + # return linkedcharts # TODO: add signalling painter system # def add_signals(self): @@ -184,8 +298,10 @@ class LinkedSplitCharts(QtGui.QWidget): zoomIsDisabled = QtCore.pyqtSignal(bool) def __init__( + self, chart_space: ChartSpace, + ) -> None: super().__init__() self.signals_visible: bool = False @@ -194,6 +310,8 @@ class LinkedSplitCharts(QtGui.QWidget): self.subplots: Dict[Tuple[str, ...], ChartPlotWidget] = {} self.chart_space = chart_space + self.chart_space = chart_space + self.xaxis = DynamicDateAxis( orientation='bottom', linked_charts=self @@ -232,6 +350,14 @@ class LinkedSplitCharts(QtGui.QWidget): sizes.extend([min_h_ind] * len(self.subplots)) self.splitter.setSizes(sizes) # , int(self.height()*0.2) + def focus(self) -> None: + if self.chart is not None: + self.chart.setFocus() + + def unfocus(self) -> None: + if self.chart is not None: + self.chart.clearFocus() + def plot_ohlc_main( self, symbol: Symbol, @@ -1159,7 +1285,7 @@ async def spawn_fsps( # XXX: fsp may have been opened by a duplicate chart. Error for # now until we figure out how to wrap fsps as "feeds". - assert opened, f"A chart for {key} likely already exists?" + # assert opened, f"A chart for {key} likely already exists?" conf['shm'] = shm @@ -1253,16 +1379,17 @@ async def run_fsp( # fsp_func_name ) - # read last value - array = shm.array - value = array[fsp_func_name][-1] + chart._lc.focus() - last_val_sticky = chart._ysticks[chart.name] - last_val_sticky.update_from_data(-1, value) + # read last value + array = shm.array + value = array[fsp_func_name][-1] - chart.update_curve_from_array(fsp_func_name, array) + last_val_sticky = chart._ysticks[chart.name] + last_val_sticky.update_from_data(-1, value) - chart._shm = shm + chart.update_curve_from_array(fsp_func_name, array) + chart._shm = shm # TODO: figure out if we can roll our own `FillToThreshold` to # get brush filled polygons for OS/OB conditions. @@ -1387,14 +1514,7 @@ async def chart_symbol( brokername: str, sym: str, loglevel: str, - task_status: TaskStatus[Symbol] = trio.TASK_STATUS_IGNORED, ) -> None: - """Spawn a real-time chart widget for this symbol and app session. - - These widgets can remain up but hidden so that multiple symbols - can be viewed and switched between extremely fast. - - """ # historical data fetch brokermod = brokers.get_brokermod(brokername) @@ -1404,24 +1524,21 @@ async def chart_symbol( loglevel=loglevel, ) as feed: - ohlcv: ShmArray = feed.shm + ohlcv = feed.shm bars = ohlcv.array symbol = feed.symbols[sym] - task_status.started(symbol) - # load in symbol's ohlc data chart_app.window.setWindowTitle( f'{symbol.key}@{symbol.brokers} ' f'tick:{symbol.tick_size}' ) - # await tractor.breakpoint() linked_charts = chart_app.linkedcharts linked_charts._symbol = symbol chart = linked_charts.plot_ohlc_main(symbol, bars) - chart.setFocus() + linked_charts.focus() # plot historical vwap if available wap_in_history = False @@ -1494,6 +1611,9 @@ async def chart_symbol( wap_in_history, ) + # await tractor.breakpoint() + # chart_app.linkedcharts.focus() + # wait for a first quote before we start any update tasks quote = await feed.receive() @@ -1514,7 +1634,7 @@ async def chart_symbol( # chart, # linked_charts, # ) - + chart_app.linkedcharts.focus() await start_order_mode(chart, symbol, brokername) @@ -1522,7 +1642,7 @@ async def _async_main( # implicit required argument provided by ``qtractor_run()`` widgets: Dict[str, Any], - symbol_key: str, + sym: str, brokername: str, loglevel: str, @@ -1550,29 +1670,24 @@ async def _async_main( # configure global DPI aware font size _font.configure_to_dpi(screen) + # try: async with trio.open_nursery() as root_n: # set root nursery for spawning other charts/feeds # that run cached in the bg chart_app._root_n = root_n - chart_app.load_symbol(brokername, symbol_key, loglevel) + # TODO: trigger on ctlr-k + chart_app.open_search() - symbol = await root_n.start( - chart_symbol, - chart_app, - brokername, - symbol_key, - loglevel, - ) - - chart_app.window.setWindowTitle( - f'{symbol.key}@{symbol.brokers} ' - f'tick:{symbol.tick_size}' - ) + # this internally starts a ``chart_symbol()`` task above + chart_app.load_symbol(brokername, sym, loglevel) await trio.sleep_forever() + # finally: + # root_n.cancel() + def _main( sym: str, diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 06258566..4c32e961 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -486,6 +486,8 @@ class ChartView(ViewBox): self._key_buffer = [] self._key_active: bool = False + self.setFocusPolicy(QtCore.Qt.StrongFocus) + @property def chart(self) -> 'ChartPlotWidget': # type: ignore # noqa return self._chart @@ -757,6 +759,12 @@ class ChartView(ViewBox): if mods == QtCore.Qt.AltModifier: pass + # ctlr-k + if key == QtCore.Qt.Key_K and ctrl: + search = self._chart._lc.chart_space.search + search.show() + search.setFocus() + # esc if key == QtCore.Qt.Key_Escape or (ctrl and key == QtCore.Qt.Key_C): # ctrl-c as cancel From 1f9f2b873abae45e99d88ff96691abf68858e644 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 13 Apr 2021 10:21:19 -0400 Subject: [PATCH 03/81] Super fast switching, just hide loaded charts --- piker/ui/_chart.py | 55 ++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index f1ac6971..8b8523b3 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -24,13 +24,11 @@ from types import ModuleType from functools import partial from PyQt5 import QtCore, QtGui -from PyQt5.QtCore import Qt from PyQt5 import QtWidgets import numpy as np import pyqtgraph as pg import tractor import trio -from trio_typing import TaskStatus from ._axes import ( DynamicDateAxis, @@ -241,15 +239,14 @@ class ChartSpace(QtGui.QWidget): # switching to a new viewable chart if not self.v_layout.isEmpty(): - # and not ( - # self.linkedcharts is linkedcharts - # ): # XXX: this is CRITICAL especially with pixel buffer caching self.linkedcharts.hide() - # remove any existing plots - self.v_layout.removeWidget(self.linkedcharts) + # XXX: pretty sure we don't need this + # remove any existing plots? + # self.v_layout.removeWidget(self.linkedcharts) + # switching to a new viewable chart if linkedcharts is None or reset: # we must load a fresh linked charts set @@ -262,24 +259,23 @@ class ChartSpace(QtGui.QWidget): loglevel, ) self.v_layout.addWidget(linkedcharts) + self._chart_cache[symbol_key] = linkedcharts - # if linkedcharts.chart: - # breakpoint() - - # else: # chart is already in memory so just focus it if self.linkedcharts: self.linkedcharts.unfocus() # self.v_layout.addWidget(linkedcharts) - self.linkedcharts = linkedcharts + linkedcharts.show() linkedcharts.focus() + self.linkedcharts = linkedcharts - # return linkedcharts - - # TODO: add signalling painter system - # def add_signals(self): - # self.chart.add_signals() + symbol = linkedcharts.symbol + if symbol is not None: + self.window.setWindowTitle( + f'{symbol.key}@{symbol.brokers} ' + f'tick:{symbol.tick_size}' + ) class LinkedSplitCharts(QtGui.QWidget): @@ -352,7 +348,7 @@ class LinkedSplitCharts(QtGui.QWidget): def focus(self) -> None: if self.chart is not None: - self.chart.setFocus() + self.chart.focus() def unfocus(self) -> None: if self.chart is not None: @@ -573,6 +569,10 @@ class ChartPlotWidget(pg.PlotWidget): # for when the splitter(s) are resized self._vb.sigResized.connect(self._set_yrange) + def focus(self) -> None: + # self.setFocus() + self._vb.setFocus() + def last_bar_in_view(self) -> int: self._ohlc[-1]['index'] @@ -1253,6 +1253,9 @@ async def spawn_fsps( Pass target entrypoint and historical data. """ + + linked_charts.focus() + # spawns sub-processes which execute cpu bound FSP code async with tractor.open_nursery(loglevel=loglevel) as n: @@ -1515,6 +1518,12 @@ async def chart_symbol( sym: str, loglevel: str, ) -> None: + """Spawn a real-time chart widget for this symbol and app session. + + These widgets can remain up but hidden so that multiple symbols + can be viewed and switched between extremely fast. + + """ # historical data fetch brokermod = brokers.get_brokermod(brokername) @@ -1533,12 +1542,12 @@ async def chart_symbol( f'{symbol.key}@{symbol.brokers} ' f'tick:{symbol.tick_size}' ) + # await tractor.breakpoint() linked_charts = chart_app.linkedcharts linked_charts._symbol = symbol chart = linked_charts.plot_ohlc_main(symbol, bars) - - linked_charts.focus() + chart.setFocus() # plot historical vwap if available wap_in_history = False @@ -1611,9 +1620,6 @@ async def chart_symbol( wap_in_history, ) - # await tractor.breakpoint() - # chart_app.linkedcharts.focus() - # wait for a first quote before we start any update tasks quote = await feed.receive() @@ -1634,7 +1640,8 @@ async def chart_symbol( # chart, # linked_charts, # ) - chart_app.linkedcharts.focus() + # chart.focus() + await start_order_mode(chart, symbol, brokername) From 157f6ab02b7e031a3f5a1289ee17dc192b7140e7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 14 Apr 2021 10:56:14 -0400 Subject: [PATCH 04/81] Drop lingering chart focus call --- piker/ui/_chart.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 8b8523b3..257a77fb 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1640,7 +1640,6 @@ async def chart_symbol( # chart, # linked_charts, # ) - # chart.focus() await start_order_mode(chart, symbol, brokername) From c26f4d9877967f43631be1fa42903feca8585a9c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 22 Apr 2021 21:23:43 -0400 Subject: [PATCH 05/81] Add kraken fuzzy symbol search --- piker/brokers/kraken.py | 73 ++++++++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index 8f83c960..c0e61a48 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -21,7 +21,7 @@ Kraken backend. from contextlib import asynccontextmanager, AsyncExitStack from dataclasses import asdict, field from types import ModuleType -from typing import List, Dict, Any, Tuple +from typing import List, Dict, Any, Tuple, Optional import json import time @@ -37,6 +37,7 @@ from trio_websocket._impl import ( import arrow import asks +from fuzzywuzzy import process as fuzzy import numpy as np import trio import tractor @@ -147,6 +148,17 @@ class Client: 'User-Agent': 'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)' }) + self._pairs = None + + @property + def pairs(self) -> Dict[str, Any]: + if self._pairs is None: + raise RuntimeError( + "Make sure to run `cache_symbols()` on startup!" + ) + # retreive and cache all symbols + + return self._pairs async def _public( self, @@ -162,14 +174,48 @@ class Client: async def symbol_info( self, - pair: str = 'all', + pair: Optional[str] = None, ): - resp = await self._public('AssetPairs', {'pair': pair}) + if pair is not None: + pairs = {'pair': pair} + else: + pairs = None # get all pairs + + resp = await self._public('AssetPairs', pairs) err = resp['error'] if err: raise BrokerError(err) - true_pair_key, data = next(iter(resp['result'].items())) - return data + + pairs = resp['result'] + + if pair is not None: + _, data = next(iter(pairs.items())) + return data + else: + return pairs + + async def cache_symbols( + self, + ) -> None: + self._pairs = await self.symbol_info() + + async def search_stocks( + self, + pattern: str, + limit: int = None, + ) -> Dict[str, Any]: + if self._pairs is not None: + data = self._pairs + else: + data = await self.symbol_info() + + matches = fuzzy.extractBests( + pattern, + data, + score_cutoff=50, + ) + # repack in dict form + return {item[0]['altname']: item[0] for item in matches} async def bars( self, @@ -232,7 +278,9 @@ class Client: @asynccontextmanager async def get_client() -> Client: - yield Client() + client = Client() + await client.cache_symbols() + yield client async def stream_messages(ws): @@ -249,7 +297,7 @@ async def stream_messages(ws): too_slow_count += 1 - if too_slow_count > 10: + if too_slow_count > 20: log.warning( "Heartbeat is too slow, resetting ws connection") @@ -368,10 +416,13 @@ class AutoReconWs: self, tries: int = 10000, ) -> None: - try: - await self._stack.aclose() - except (DisconnectionTimeout, RuntimeError): - await trio.sleep(1) + while True: + try: + await self._stack.aclose() + except (DisconnectionTimeout, RuntimeError): + await trio.sleep(1) + else: + break last_err = None for i in range(tries): From 0e83906f1180f6ea7f769722b4504568d1a4b647 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 23 Apr 2021 11:12:29 -0400 Subject: [PATCH 06/81] Initial WIP search completer; still a mess --- piker/ui/_chart.py | 133 +++----------- piker/ui/_search.py | 415 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 435 insertions(+), 113 deletions(-) create mode 100644 piker/ui/_search.py diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 257a77fb..5adc832c 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -24,7 +24,6 @@ from types import ModuleType from functools import partial from PyQt5 import QtCore, QtGui -from PyQt5 import QtWidgets import numpy as np import pyqtgraph as pg import tractor @@ -48,7 +47,6 @@ from ._graphics._ohlc import BarItems from ._graphics._curve import FastAppendCurve from ._style import ( _font, - DpiAwareFont, hcolor, CHART_MARGINS, _xaxis_at, @@ -56,6 +54,7 @@ from ._style import ( _bars_from_right_in_follow_mode, _bars_to_left_in_follow_mode, ) +from ._search import FontSizedQLineEdit from ..data._source import Symbol from ..data._sharedmem import ShmArray from .. import brokers @@ -71,99 +70,6 @@ from .. import fsp log = get_logger(__name__) -class FontSizedQLineEdit(QtWidgets.QLineEdit): - - def __init__( - self, - parent_chart: 'ChartSpace', - font: DpiAwareFont = _font, - ) -> None: - super().__init__(parent_chart) - - self.dpi_font = font - self.chart_app = parent_chart - - # size it as we specify - self.setSizePolicy( - QtWidgets.QSizePolicy.Fixed, - QtWidgets.QSizePolicy.Fixed, - ) - self.setFont(font.font) - - # witty bit of margin - self.setTextMargins(2, 2, 2, 2) - - def sizeHint(self) -> QtCore.QSize: - psh = super().sizeHint() - psh.setHeight(self.dpi_font.px_size + 2) - return psh - - def unfocus(self) -> None: - self.hide() - self.clearFocus() - - def keyPressEvent(self, ev: QtCore.QEvent) -> None: - # by default we don't markt it as consumed? - # ev.ignore() - super().keyPressEvent(ev) - - ev.accept() - # text = ev.text() - key = ev.key() - mods = ev.modifiers() - - ctrl = False - if mods == QtCore.Qt.ControlModifier: - ctrl = True - - if ctrl: - if key == QtCore.Qt.Key_C: - self.unfocus() - self.chart_app.linkedcharts.focus() - - # TODO: - elif key == QtCore.Qt.Key_K: - print('move up') - - elif key == QtCore.Qt.Key_J: - print('move down') - - elif key in (QtCore.Qt.Key_Enter, QtCore.Qt.Key_Return): - print(f'Requesting symbol: {self.text()}') - symbol = self.text() - app = self.chart_app - self.chart_app.load_symbol( - app.linkedcharts.symbol.brokers[0], - symbol, - 'info', - ) - # self.hide() - self.unfocus() - - # if self._executing(): - # # ignore all key presses while executing, except for Ctrl-C - # if event.modifiers() == Qt.ControlModifier and key == Qt.Key_C: - # self._handle_ctrl_c() - # return True - - # handler = self._key_event_handlers.get(key) - # intercepted = handler and handler(event) - - # Assumes that Control+Key is a movement command, i.e. should not be - # handled as text insertion. However, on win10 AltGr is reported as - # Alt+Control which is why we handle this case like regular - # # keypresses, see #53: - # if not event.modifiers() & Qt.ControlModifier or \ - # event.modifiers() & Qt.AltModifier: - # self._keep_cursor_in_buffer() - - # if not intercepted and event.text(): - # intercepted = True - # self.insert_input_text(event.text()) - - # return False - - class ChartSpace(QtGui.QWidget): """High level widget which contains layouts for organizing lower level charts as well as other widgets used to control @@ -172,9 +78,9 @@ class ChartSpace(QtGui.QWidget): def __init__(self, parent=None): super().__init__(parent) - self.v_layout = QtGui.QVBoxLayout(self) - self.v_layout.setContentsMargins(0, 0, 0, 0) - self.v_layout.setSpacing(1) + self.vbox = QtGui.QVBoxLayout(self) + self.vbox.setContentsMargins(0, 0, 0, 0) + self.vbox.setSpacing(2) self.toolbar_layout = QtGui.QHBoxLayout() self.toolbar_layout.setContentsMargins(0, 0, 0, 0) @@ -184,8 +90,8 @@ class ChartSpace(QtGui.QWidget): # self.init_timeframes_ui() # self.init_strategy_ui() - self.v_layout.addLayout(self.toolbar_layout) - self.v_layout.addLayout(self.h_layout) + self.vbox.addLayout(self.toolbar_layout) + self.vbox.addLayout(self.h_layout) self._chart_cache = {} self.linkedcharts: 'LinkedSplitCharts' = None self.symbol_label: Optional[QtGui.QLabel] = None @@ -196,7 +102,16 @@ class ChartSpace(QtGui.QWidget): # search = self.search = QtWidgets.QLineEdit() self.search = FontSizedQLineEdit(self) self.search.unfocus() - self.v_layout.addWidget(self.search) + self.vbox.addWidget(self.search) + self.vbox.addWidget(self.search.view) + self.search.view.set_results([ + 'XMRUSD', + 'XBTUSD', + 'XMRXBT', + # 'XMRXBT', + # 'XDGUSD', + # 'ADAUSD', + ]) # search.installEventFilter(self) @@ -237,14 +152,13 @@ class ChartSpace(QtGui.QWidget): """ linkedcharts = self._chart_cache.get(symbol_key) - # switching to a new viewable chart - if not self.v_layout.isEmpty(): + if not self.vbox.isEmpty(): # XXX: this is CRITICAL especially with pixel buffer caching self.linkedcharts.hide() # XXX: pretty sure we don't need this # remove any existing plots? - # self.v_layout.removeWidget(self.linkedcharts) + # self.vbox.removeWidget(self.linkedcharts) # switching to a new viewable chart if linkedcharts is None or reset: @@ -258,14 +172,14 @@ class ChartSpace(QtGui.QWidget): symbol_key, loglevel, ) - self.v_layout.addWidget(linkedcharts) + self.vbox.addWidget(linkedcharts) self._chart_cache[symbol_key] = linkedcharts # chart is already in memory so just focus it if self.linkedcharts: self.linkedcharts.unfocus() - # self.v_layout.addWidget(linkedcharts) + # self.vbox.addWidget(linkedcharts) linkedcharts.show() linkedcharts.focus() self.linkedcharts = linkedcharts @@ -1348,7 +1262,6 @@ async def run_fsp( # data-array as first msg _ = await stream.receive() - conf['stream'] = stream conf['portal'] = portal shm = conf['shm'] @@ -1414,8 +1327,6 @@ async def run_fsp( chart._set_yrange() - stream = conf['stream'] - last = time.time() # update chart graphics @@ -1676,7 +1587,6 @@ async def _async_main( # configure global DPI aware font size _font.configure_to_dpi(screen) - # try: async with trio.open_nursery() as root_n: # set root nursery for spawning other charts/feeds @@ -1691,9 +1601,6 @@ async def _async_main( await trio.sleep_forever() - # finally: - # root_n.cancel() - def _main( sym: str, diff --git a/piker/ui/_search.py b/piker/ui/_search.py new file mode 100644 index 00000000..02929041 --- /dev/null +++ b/piker/ui/_search.py @@ -0,0 +1,415 @@ +# piker: trading gear for hackers +# Copyright (C) Tyler Goodlet (in stewardship for piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +""" +qompleterz: embeddable search and complete using trio, Qt and fuzzywuzzy. + +""" +from typing import Dict, List, Optional +import sys + +from PyQt5 import QtCore, QtGui +from PyQt5.QtCore import Qt +from PyQt5 import QtWidgets + +from PyQt5.QtCore import ( + Qt, + QSize, + QModelIndex, + QItemSelectionModel, + # QStringListModel +) +from PyQt5.QtGui import ( + QStandardItem, + QStandardItemModel, +) +from PyQt5.QtWidgets import ( + QTreeView, + # QListWidgetItem, + QAbstractScrollArea, + QStyledItemDelegate, +) +from fuzzywuzzy import process + + +# from PyQt5.QtWidgets import QCompleter, QComboBox + +from ..log import get_logger +from ._style import ( + _font, + DpiAwareFont, + hcolor, +) + + +log = get_logger(__name__) + + +class SimpleDelegate(QStyledItemDelegate): + """ + Super simple view delegate to render text in the same + font size as the search widget. + + """ + + def __init__( + self, + parent=None, + font: DpiAwareFont = _font, + ) -> None: + super().__init__(parent) + self.dpi_font = font + + # def sizeHint(self, *args) -> QtCore.QSize: + # """ + # Scale edit box to size of dpi aware font. + + # """ + # psh = super().sizeHint(*args) + # # psh.setHeight(self.dpi_font.px_size + 2) + + # psh.setHeight(18) + # # psh.setHeight(18) + # return psh + + +class CompleterView(QTreeView): + + def __init__( + self, + parent=None, + ) -> None: + + super().__init__(parent) + + self._font_size: int = 0 # pixels + self._cache: Dict[str, List[str]] = {} + + # def viewportSizeHint(self) -> QtCore.QSize: + # vps = super().viewportSizeHint() + # return QSize(vps.width(), _font.px_size * 6 * 2) + + # def sizeHint(self) -> QtCore.QSize: + # """Scale completion results up to 6/16 of window. + # """ + # # height = self.window().height() * 1/6 + # # psh.setHeight(self.dpi_font.px_size * 6) + # # print(_font.px_size) + # height = _font.px_size * 6 * 2 + # # the default here is just the vp size without scroll bar + # # https://doc.qt.io/qt-5/qabstractscrollarea.html#viewportSizeHint + # vps = self.viewportSizeHint() + # # print(f'h: {height}\n{vps}') + # # psh.setHeight(12) + # return QSize(-1, height) + + def resize(self): + model = self.model() + cols = model.columnCount() + + for i in range(cols): + self.resizeColumnToContents(i) + + # inclusive of search bar and header "rows" in pixel terms + rows = model.rowCount() + 2 + # max_rows = 8 # 6 + search and headers + row_px = self.rowHeight(self.currentIndex()) + # print(f'font_h: {font_h}\n px_height: {px_height}') + + self.setMinimumSize(self.width(), rows * row_px) + self.setMaximumSize(self.width(), rows * row_px) + + def set_font_size(self, size: int = 18): + # dpi_px_size = _font.px_size + print(size) + if size < 0: + size = 16 + + self._font_size = size + + self.setStyleSheet(f"font: {size}px") + + def set_results( + self, + results: List[str], + ) -> None: + + model = self.model() + + for i, s in enumerate(results): + + ix = QStandardItem(str(i)) + item = QStandardItem(s) + # item.setCheckable(False) + + src = QStandardItem('kraken') + + # Add the item to the model + model.appendRow([src, ix, item]) + + def find_matches( + self, + field: str, + txt: str, + ) -> List[QStandardItem]: + model = self.model() + items = model.findItems( + txt, + Qt.MatchContains, + self.field_to_col(field), + ) + + +def mk_completer_view( + + labels: List[str], + +) -> QTreeView: + + tree = CompleterView() + model = QStandardItemModel(tree) + + # a std "tabular" config + tree.setItemDelegate(SimpleDelegate()) + tree.setModel(model) + tree.setAlternatingRowColors(True) + tree.setIndentation(1) + + # tree.setUniformRowHeights(True) + # tree.setColumnWidth(0, 3) + + # ux settings + tree.setItemsExpandable(True) + tree.setExpandsOnDoubleClick(False) + tree.setAnimated(False) + tree.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) + + tree.setSizeAdjustPolicy(QAbstractScrollArea.AdjustIgnored) + + # column headers + model.setHorizontalHeaderLabels(labels) + + return tree + + +class FontSizedQLineEdit(QtWidgets.QLineEdit): + + def __init__( + self, + parent_chart: 'ChartSpace', # noqa + view: Optional[CompleterView] = None, + font: DpiAwareFont = _font, + ) -> None: + super().__init__(parent_chart) + + # vbox = self.vbox = QtGui.QVBoxLayout(self) + # vbox.addWidget(self) + # self.vbox.setContentsMargins(0, 0, 0, 0) + # self.vbox.setSpacing(2) + + self._view: CompleterView = view + + self.dpi_font = font + self.chart_app = parent_chart + + # size it as we specify + self.setSizePolicy( + QtWidgets.QSizePolicy.Fixed, + QtWidgets.QSizePolicy.Fixed, + ) + self.setFont(font.font) + + # witty bit of margin + self.setTextMargins(2, 2, 2, 2) + + # self.setContextMenuPolicy(Qt.CustomContextMenu) + # self.customContextMenuRequested.connect(self.show_menu) + # self.setStyleSheet(f"font: 18px") + + def focus(self) -> None: + self.clear() + self.show() + self.setFocus() + + def show(self) -> None: + super().show() + self.show_matches() + # self.view.show() + # self.view.resize() + + @property + def view(self) -> CompleterView: + + if self._view is None: + view = mk_completer_view(['src', 'i', 'symbol']) + + # print('yo') + # self.chart_app.vbox.addWidget(view) + # self.vbox.addWidget(view) + + self._view = view + + return self._view + + def show_matches(self): + view = self.view + view.set_font_size(self.dpi_font.px_size) + view.show() + # scale columns + view.resize() + + def sizeHint(self) -> QtCore.QSize: + """ + Scale edit box to size of dpi aware font. + + """ + psh = super().sizeHint() + psh.setHeight(self.dpi_font.px_size + 2) + # psh.setHeight(12) + return psh + + def unfocus(self) -> None: + self.hide() + self.clearFocus() + + if self.view: + self.view.hide() + + def keyPressEvent(self, ev: QtCore.QEvent) -> None: + # by default we don't markt it as consumed? + # ev.ignore() + super().keyPressEvent(ev) + + ev.accept() + # text = ev.text() + key = ev.key() + mods = ev.modifiers() + txt = self.text() + + if key in (Qt.Key_Enter, Qt.Key_Return): + + print(f'Requesting symbol: {self.text()}') + + # TODO: ensure there is a matching completion or error and + # do nothing + symbol = txt + + app = self.chart_app + self.chart_app.load_symbol( + app.linkedcharts.symbol.brokers[0], + symbol, + 'info', + ) + self.unfocus() + return + + ctrl = False + if mods == Qt.ControlModifier: + ctrl = True + + view = self.view + model = view.model() + nidx = cidx = view.currentIndex() + sel = view.selectionModel() + # sel.clear() + + # selection tips: + # - get parent: self.index(row, 0) + # - first item index: index = self.index(0, 0, parent) + + if ctrl: + # we're in select mode or cancelling + + if key == Qt.Key_C: + self.unfocus() + + # kill the search and focus back on main chart + if self.chart_app: + self.chart_app.linkedcharts.focus() + + return + + # result selection nav + if key in (Qt.Key_K, Qt.Key_J): + + if key == Qt.Key_K: + # self.view.setFocus() + nidx = view.indexAbove(cidx) + print('move up') + + elif key == Qt.Key_J: + # self.view.setFocus() + nidx = view.indexBelow(cidx) + print('move down') + + # select row without selecting.. :eye_rollzz: + # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex + if nidx.isValid(): + sel.setCurrentIndex( + nidx, + QItemSelectionModel.ClearAndSelect | + QItemSelectionModel.Rows + ) + + # TODO: make this not hard coded to 2 + # and use the ``CompleterView`` schema/settings + # to figure out the desired field(s) + value = model.item(nidx.row(), 2).text() + print(f'value: {value}') + self.setText(value) + + else: + sel.setCurrentIndex( + model.index(0, 0, QModelIndex()), + QItemSelectionModel.ClearAndSelect | # type: ignore[arg-type] + QItemSelectionModel.Rows + ) + + self.show_matches() + + +if __name__ == '__main__': + + # local testing of **just** the search UI + app = QtWidgets.QApplication(sys.argv) + + syms = [ + 'XMRUSD', + 'XBTUSD', + 'XMRXBT', + 'XDGUSD', + 'ADAUSD', + ] + + # results.setFocusPolicy(Qt.NoFocus) + search = FontSizedQLineEdit(None, view=view) + search.view.set_results(syms) + + # make a root widget to tie shit together + class W(QtGui.QWidget): + def __init__(self, parent=None): + super().__init__(parent) + vbox = self.vbox = QtGui.QVBoxLayout(self) + self.vbox.setContentsMargins(0, 0, 0, 0) + self.vbox.setSpacing(2) + + main = W() + main.vbox.addWidget(search) + main.vbox.addWidget(view) + # main.show() + search.show() + + sys.exit(app.exec_()) From da0cb9b2acdd1718e404dce65e1f8cba4b9538f2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 23 Apr 2021 11:13:23 -0400 Subject: [PATCH 07/81] Call search focus directly --- piker/ui/_interaction.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 4c32e961..4a9771e4 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -762,8 +762,7 @@ class ChartView(ViewBox): # ctlr-k if key == QtCore.Qt.Key_K and ctrl: search = self._chart._lc.chart_space.search - search.show() - search.setFocus() + search.focus() # esc if key == QtCore.Qt.Key_Escape or (ctrl and key == QtCore.Qt.Key_C): From 821d5ab9ec7f4192d828d47479802543e12a41d4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 29 Apr 2021 09:03:28 -0400 Subject: [PATCH 08/81] Bring back in and merge tractor stream api patch --- piker/ui/_chart.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 5adc832c..eb868c01 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1262,8 +1262,6 @@ async def run_fsp( # data-array as first msg _ = await stream.receive() - conf['portal'] = portal - shm = conf['shm'] if conf.get('overlay'): @@ -1305,6 +1303,9 @@ async def run_fsp( last_val_sticky.update_from_data(-1, value) chart.update_curve_from_array(fsp_func_name, array) + + chart._shm = shm + chart.update_curve_from_array(fsp_func_name, shm.array) chart._shm = shm # TODO: figure out if we can roll our own `FillToThreshold` to @@ -1347,7 +1348,6 @@ async def run_fsp( # re-compute steps. read_tries = 2 while read_tries > 0: - try: # read last array = shm.array @@ -1444,7 +1444,7 @@ async def chart_symbol( loglevel=loglevel, ) as feed: - ohlcv = feed.shm + ohlcv: ShmArray = feed.shm bars = ohlcv.array symbol = feed.symbols[sym] From 5e1b15f3190bdc426d8fb1b0d4c7153f436bed76 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 29 Apr 2021 15:23:16 -0400 Subject: [PATCH 09/81] Repair indents from rebasing --- piker/ui/_chart.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index eb868c01..6b6e4f83 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1293,20 +1293,27 @@ async def run_fsp( # fsp_func_name ) + # XXX: ONLY for sub-chart fsps, overlays have their + # data looked up from the chart's internal array set. + # TODO: we must get a data view api going STAT!! + chart._shm = shm + + # should **not** be the same sub-chart widget + assert chart.name != linked_charts.chart.name + + # sticky only on sub-charts atm + last_val_sticky = chart._ysticks[chart.name] + + # read from last calculated value + array = shm.array + value = array[fsp_func_name][-1] + last_val_sticky.update_from_data(-1, value) + chart._lc.focus() - # read last value - array = shm.array - value = array[fsp_func_name][-1] - - last_val_sticky = chart._ysticks[chart.name] - last_val_sticky.update_from_data(-1, value) - - chart.update_curve_from_array(fsp_func_name, array) - - chart._shm = shm + # works also for overlays in which case data is looked up from + # internal chart array set.... chart.update_curve_from_array(fsp_func_name, shm.array) - chart._shm = shm # TODO: figure out if we can roll our own `FillToThreshold` to # get brush filled polygons for OS/OB conditions. From 2861f321ce43f7470d902420ef8bb0d5d2f68166 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 5 May 2021 10:09:25 -0400 Subject: [PATCH 10/81] Add async keyboard cloner sub-sys --- piker/ui/_event.py | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 piker/ui/_event.py diff --git a/piker/ui/_event.py b/piker/ui/_event.py new file mode 100644 index 00000000..889f4e34 --- /dev/null +++ b/piker/ui/_event.py @@ -0,0 +1,77 @@ +# piker: trading gear for hackers +# Copyright (C) Tyler Goodlet (in stewardship for piker0) + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +""" +Qt event proxying and processing using ``trio`` mem chans. + +""" +from contextlib import asynccontextmanager + +from PyQt5 import QtCore, QtGui +from PyQt5.QtCore import QEvent +import trio + + +class KeyCloner(QtCore.QObject): + """Clone and forward keyboard events over a trio memory channel + for later async processing. + + """ + + _send_chan: trio.abc.SendChannel = None + + def eventFilter( + self, + source: QtGui.QWidget, + ev: QEvent, + ) -> None: + + if ev.type() == QEvent.KeyPress: + + # XXX: we unpack here because apparently doing it + # after pop from the mem chan isn't showing the same + # event object? no clue wtf is going on there, likely + # something to do with Qt internals and calling the + # parent handler? + key = ev.key() + mods = ev.modifiers() + txt = ev.text() + + # run async processing + self._send_chan.send_nowait((key, mods, txt)) + + # never intercept the event + return False + + +@asynccontextmanager +async def open_key_stream( + + source_widget: QtGui.QWidget, + +) -> trio.abc.ReceiveChannel: + + send, recv = trio.open_memory_channel(1) + + kc = KeyCloner() + kc._send_chan = send + source_widget.installEventFilter(kc) + + try: + yield recv + finally: + await send.aclose() + source_widget.removeEventFilter(kc) From 60d44f30ee43e94eccc8aaa8bf6ece1acd832636 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 5 May 2021 10:10:02 -0400 Subject: [PATCH 11/81] Make search kb handling async --- piker/ui/_search.py | 73 ++++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 02929041..3a6d4d6e 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -18,12 +18,13 @@ qompleterz: embeddable search and complete using trio, Qt and fuzzywuzzy. """ -from typing import Dict, List, Optional +from typing import List, Optional import sys from PyQt5 import QtCore, QtGui -from PyQt5.QtCore import Qt +from PyQt5.QtCore import Qt, QEvent from PyQt5 import QtWidgets +import trio from PyQt5.QtCore import ( Qt, @@ -45,8 +46,6 @@ from PyQt5.QtWidgets import ( from fuzzywuzzy import process -# from PyQt5.QtWidgets import QCompleter, QComboBox - from ..log import get_logger from ._style import ( _font, @@ -96,7 +95,7 @@ class CompleterView(QTreeView): super().__init__(parent) self._font_size: int = 0 # pixels - self._cache: Dict[str, List[str]] = {} + # self._cache: Dict[str, List[str]] = {} # def viewportSizeHint(self) -> QtCore.QSize: # vps = super().viewportSizeHint() @@ -129,6 +128,7 @@ class CompleterView(QTreeView): row_px = self.rowHeight(self.currentIndex()) # print(f'font_h: {font_h}\n px_height: {px_height}') + # TODO: probably make this more general / less hacky self.setMinimumSize(self.width(), rows * row_px) self.setMaximumSize(self.width(), rows * row_px) @@ -288,16 +288,36 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): if self.view: self.view.hide() - def keyPressEvent(self, ev: QtCore.QEvent) -> None: - # by default we don't markt it as consumed? - # ev.ignore() - super().keyPressEvent(ev) + # def keyPressEvent(self, ev: QEvent) -> None: - ev.accept() - # text = ev.text() - key = ev.key() - mods = ev.modifiers() - txt = self.text() + # # XXX: we unpack here because apparently doing it + # # after pop from the mem chan isn't showing the same + # # event object? no clue wtf is going on there, likely + # # something to do with Qt internals and calling the + # # parent handler? + # key = ev.key() + # mods = ev.modifiers() + # txt = self.text() + + # # run async processing + # self._send_chan.send_nowait((key, mods, txt)) + + # super().keyPressEvent(ev) + + # # ev.accept() + + +async def handle_keyboard_input( + self, + recv_chan: trio.abc.ReceiveChannel, +) -> None: + + async for key, mods, txt in recv_chan: + + # by default we don't mart it as consumed? + # ev.ignore() + + print(f'key: {key}, mods: {mods}, txt: {txt}') if key in (Qt.Key_Enter, Qt.Key_Return): @@ -305,7 +325,7 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): # TODO: ensure there is a matching completion or error and # do nothing - symbol = txt + symbol = self.text() app = self.chart_app self.chart_app.load_symbol( @@ -313,8 +333,11 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): symbol, 'info', ) + + # release kb control of search bar self.unfocus() - return + continue + # return ctrl = False if mods == Qt.ControlModifier: @@ -340,7 +363,8 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): if self.chart_app: self.chart_app.linkedcharts.focus() - return + # return + continue # result selection nav if key in (Qt.Key_K, Qt.Key_J): @@ -364,12 +388,12 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): QItemSelectionModel.Rows ) - # TODO: make this not hard coded to 2 - # and use the ``CompleterView`` schema/settings - # to figure out the desired field(s) - value = model.item(nidx.row(), 2).text() - print(f'value: {value}') - self.setText(value) + # TODO: make this not hard coded to 2 + # and use the ``CompleterView`` schema/settings + # to figure out the desired field(s) + value = model.item(nidx.row(), 2).text() + print(f'value: {value}') + self.setText(value) else: sel.setCurrentIndex( @@ -395,6 +419,7 @@ if __name__ == '__main__': ] # results.setFocusPolicy(Qt.NoFocus) + view = mk_completer_view(['src', 'i', 'symbol']) search = FontSizedQLineEdit(None, view=view) search.view.set_results(syms) @@ -402,7 +427,7 @@ if __name__ == '__main__': class W(QtGui.QWidget): def __init__(self, parent=None): super().__init__(parent) - vbox = self.vbox = QtGui.QVBoxLayout(self) + self.vbox = QtGui.QVBoxLayout(self) self.vbox.setContentsMargins(0, 0, 0, 0) self.vbox.setSpacing(2) From be39e9bdf5cab4f816a34d4277658e15623fdf3a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 5 May 2021 10:10:34 -0400 Subject: [PATCH 12/81] Load async kb search handler at startup --- piker/ui/_chart.py | 64 +++++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 6b6e4f83..0057ac10 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -54,7 +54,9 @@ from ._style import ( _bars_from_right_in_follow_mode, _bars_to_left_in_follow_mode, ) +from . import _search from ._search import FontSizedQLineEdit +from ._event import open_key_stream from ..data._source import Symbol from ..data._sharedmem import ShmArray from .. import brokers @@ -85,36 +87,19 @@ class ChartSpace(QtGui.QWidget): self.toolbar_layout = QtGui.QHBoxLayout() self.toolbar_layout.setContentsMargins(0, 0, 0, 0) - self.h_layout = QtGui.QHBoxLayout() - self.h_layout.setContentsMargins(0, 0, 0, 0) + self.hbox = QtGui.QHBoxLayout() + self.hbox.setContentsMargins(0, 0, 0, 0) # self.init_timeframes_ui() # self.init_strategy_ui() self.vbox.addLayout(self.toolbar_layout) - self.vbox.addLayout(self.h_layout) + self.vbox.addLayout(self.hbox) self._chart_cache = {} self.linkedcharts: 'LinkedSplitCharts' = None self.symbol_label: Optional[QtGui.QLabel] = None self._root_n: Optional[trio.Nursery] = None - def open_search(self): - # search = self.search = QtWidgets.QLineEdit() - self.search = FontSizedQLineEdit(self) - self.search.unfocus() - self.vbox.addWidget(self.search) - self.vbox.addWidget(self.search.view) - self.search.view.set_results([ - 'XMRUSD', - 'XBTUSD', - 'XMRXBT', - # 'XMRXBT', - # 'XDGUSD', - # 'ADAUSD', - ]) - - # search.installEventFilter(self) - def init_timeframes_ui(self): self.tf_layout = QtGui.QHBoxLayout() self.tf_layout.setSpacing(0) @@ -165,6 +150,8 @@ class ChartSpace(QtGui.QWidget): # we must load a fresh linked charts set linkedcharts = LinkedSplitCharts(self) + + # spawn new task to start up and update new sub-chart instances self._root_n.start_soon( chart_symbol, self, @@ -172,6 +159,7 @@ class ChartSpace(QtGui.QWidget): symbol_key, loglevel, ) + self.vbox.addWidget(linkedcharts) self._chart_cache[symbol_key] = linkedcharts @@ -1600,13 +1588,43 @@ async def _async_main( # that run cached in the bg chart_app._root_n = root_n - # TODO: trigger on ctlr-k - chart_app.open_search() + # setup search widget + # search.installEventFilter(self) + + search = _search.FontSizedQLineEdit(chart_app) + # the main chart's view is given focus at startup + search.unfocus() + + # add search singleton to global chart-space widget + chart_app.vbox.addWidget(search) + chart_app.vbox.addWidget(search.view) + chart_app.search = search + + search.view.set_results([ + 'ETHUSD', + 'XMRUSD', + 'XBTUSD', + 'XMRXBT', + # 'XMRXBT', + # 'XDGUSD', + # 'ADAUSD', + ]) # this internally starts a ``chart_symbol()`` task above chart_app.load_symbol(brokername, sym, loglevel) - await trio.sleep_forever() + async with open_key_stream( + search, + ) as key_stream: + + # start kb handling task for searcher + root_n.start_soon( + _search.handle_keyboard_input, + search, + key_stream, + ) + + await trio.sleep_forever() def _main( From 4b818ea2f2b5719f6630b0a686226bbc52af71db Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 5 May 2021 10:12:23 -0400 Subject: [PATCH 13/81] Add initial symbol search api for kraken --- piker/brokers/kraken.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index c0e61a48..c2db0be9 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -196,8 +196,11 @@ class Client: async def cache_symbols( self, - ) -> None: - self._pairs = await self.symbol_info() + ) -> dict: + if not self._pairs: + self._pairs = await self.symbol_info() + + return self._pairs async def search_stocks( self, @@ -279,10 +282,38 @@ class Client: @asynccontextmanager async def get_client() -> Client: client = Client() + + # load all symbols locally for fast search await client.cache_symbols() + yield client +async def open_symbol_search( + ctx: tractor.Context, +) -> Client: + async with open_cached_client('kraken') as client: + + # load all symbols locally for fast search + cache = await client.cache_symbols() + await ctx.started(cache) + + async with ctx.open_stream() as stream: + + async for pattern in stream: + + matches = fuzzy.extractBests( + pattern, + cache, + score_cutoff=50, + ) + # repack in dict form + await stream.send( + {item[0]['altname']: item[0] + for item in matches} + ) + + async def stream_messages(ws): too_slow_count = last_hb = 0 From 534553a6f59d97c9662121675e4edef09c972949 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 6 May 2021 16:37:23 -0400 Subject: [PATCH 14/81] Add client side multi-provider feed symbol search --- piker/data/feed.py | 102 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 12 deletions(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index 76c2bc23..b9df8595 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -17,6 +17,8 @@ """ Data feed apis and infra. +This module is enabled for ``brokerd`` daemons. + """ from dataclasses import dataclass, field from contextlib import asynccontextmanager @@ -25,13 +27,14 @@ from types import ModuleType from typing import ( Dict, Any, Sequence, AsyncIterator, Optional, - List + List, Awaitable, Callable ) import trio from trio_typing import TaskStatus import tractor from pydantic import BaseModel +from fuzzywuzzy import process as fuzzy from ..brokers import get_brokermod from ..log import get_logger, get_console_log @@ -43,6 +46,7 @@ from ._sharedmem import ( attach_shm_array, ShmArray, ) +from .ingest import get_ingestormod from ._source import base_iohlc_dtype, Symbol from ._sampling import ( _shms, @@ -51,7 +55,6 @@ from ._sampling import ( iter_ohlc_periods, sample_and_broadcast, ) -from .ingest import get_ingestormod log = get_logger(__name__) @@ -172,7 +175,7 @@ async def allocate_persistent_feed( ) # do history validation? - assert opened, f'Persistent shm for {symbol} was already open?!' + # assert opened, f'Persistent shm for {symbol} was already open?!' # if not opened: # raise RuntimeError("Persistent shm for sym was already open?!") @@ -235,6 +238,7 @@ async def allocate_persistent_feed( @tractor.stream async def attach_feed_bus( + ctx: tractor.Context, brokername: str, symbol: str, @@ -313,6 +317,8 @@ class Feed: _trade_stream: Optional[AsyncIterator[Dict[str, Any]]] = None _max_sample_rate: int = 0 + search: Callable[..., Awaitable] = None + # cache of symbol info messages received as first message when # a stream startsc. symbols: Dict[str, Symbol] = field(default_factory=dict) @@ -335,6 +341,7 @@ class Feed: iter_ohlc_periods, delay_s=delay_s or self._max_sample_rate, ) as self._index_stream: + yield self._index_stream else: yield self._index_stream @@ -366,8 +373,29 @@ class Feed: ) as self._trade_stream: yield self._trade_stream else: + yield self._trade_stream + @asynccontextmanager + async def open_symbol_search(self) -> AsyncIterator[dict]: + + async with self._brokerd_portal.open_context( + + self.mod.open_symbol_search, + + ) as (ctx, cache): + + async with ctx.open_stream() as stream: + + async def search(text: str) -> Dict[str, Any]: + await stream.send(text) + return await stream.receive() + + # deliver search func to consumer + self.search = search + yield search + self.search = None + def sym_to_shm_key( broker: str, @@ -376,6 +404,39 @@ def sym_to_shm_key( return f'{broker}.{symbol}' +# cache of brokernames to feeds +_cache: Dict[str, Feed] = {} +_cache_lock: trio.Lock = trio.Lock() + + +def get_multi_search() -> Callable[..., Awaitable]: + + global _cache + + async def multisearcher( + pattern: str, + ) -> dict: + + matches = {} + + async def pack_matches( + brokername: str, + pattern: str, + ) -> None: + matches[brokername] = await feed.search(pattern) + + # TODO: make this an async stream? + async with trio.open_nursery() as n: + + for (brokername, startup_sym), feed in _cache.items(): + if feed.search: + n.start_soon(pack_matches, brokername, pattern) + + return matches + + return multisearcher + + @asynccontextmanager async def open_feed( brokername: str, @@ -383,20 +444,34 @@ async def open_feed( loglevel: Optional[str] = None, ) -> AsyncIterator[Dict[str, Any]]: """Open a "data feed" which provides streamed real-time quotes. + """ + global _cache, _cache_lock + + sym = symbols[0] + + # TODO: feed cache locking, right now this is causing + # issues when reconncting to a long running emsd? + + # async with _cache_lock: + # feed = _cache.get((brokername, sym)) + + # # if feed is not None and sym in feed.symbols: + # if feed is not None: + # yield feed + # # short circuit + # return + try: mod = get_brokermod(brokername) except ImportError: mod = get_ingestormod(brokername) - if loglevel is None: - loglevel = tractor.current_actor().loglevel - - # TODO: do all! - sym = symbols[0] - - # TODO: compress these to one line with py3.9+ - async with maybe_spawn_brokerd(brokername, loglevel=loglevel) as portal: + # no feed for broker exists so maybe spawn a data brokerd + async with maybe_spawn_brokerd( + brokername, + loglevel=loglevel + ) as portal: async with portal.open_stream_from( @@ -449,8 +524,11 @@ async def open_feed( feed._max_sample_rate = max(ohlc_sample_rates) + _cache[(brokername, sym)] = feed + try: - yield feed + async with feed.open_symbol_search(): + yield feed finally: # always cancel the far end producer task From a5826e6e220169f91320222685a997ea7631aefe Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 6 May 2021 16:37:56 -0400 Subject: [PATCH 15/81] Up the kb event queue size --- piker/ui/_event.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/piker/ui/_event.py b/piker/ui/_event.py index 889f4e34..3d056d43 100644 --- a/piker/ui/_event.py +++ b/piker/ui/_event.py @@ -64,7 +64,8 @@ async def open_key_stream( ) -> trio.abc.ReceiveChannel: - send, recv = trio.open_memory_channel(1) + # 1 to force eager sending + send, recv = trio.open_memory_channel(16) kc = KeyCloner() kc._send_chan = send From ddeb9e7a946946614974771b7a90e2f64ef31889 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 6 May 2021 16:38:18 -0400 Subject: [PATCH 16/81] Add fuzzywuzzy as dep --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index fe68640e..4b8d8b1d 100755 --- a/setup.py +++ b/setup.py @@ -71,9 +71,11 @@ setup( 'PyQt5', 'pyqtgraph', 'qdarkstyle==2.8.1', + #'kivy', see requirement.txt; using a custom branch atm # tsdbs 'pymarketstore', + #'kivy', see requirement.txt; using a custom branch atm # fuzzy search From ad494db21307d5c08e7b8adcb51b2446c7889caa Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 6 May 2021 16:39:29 -0400 Subject: [PATCH 17/81] Make search routine a tractor context --- piker/brokers/kraken.py | 1 + 1 file changed, 1 insertion(+) diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index c2db0be9..0dbab324 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -289,6 +289,7 @@ async def get_client() -> Client: yield client +@tractor.context async def open_symbol_search( ctx: tractor.Context, ) -> Client: From 431fdd3f9c9737b720434edb0aa4d9b98cd9adf1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 6 May 2021 16:41:15 -0400 Subject: [PATCH 18/81] Add initial working symbol search with async completions --- piker/ui/_search.py | 370 ++++++++++++++++++++++++++------------------ 1 file changed, 217 insertions(+), 153 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 3a6d4d6e..f8681f76 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -18,20 +18,19 @@ qompleterz: embeddable search and complete using trio, Qt and fuzzywuzzy. """ -from typing import List, Optional +from typing import List, Optional, Callable, Awaitable import sys +# from pprint import pformat from PyQt5 import QtCore, QtGui -from PyQt5.QtCore import Qt, QEvent from PyQt5 import QtWidgets import trio from PyQt5.QtCore import ( Qt, - QSize, + # QSize, QModelIndex, QItemSelectionModel, - # QStringListModel ) from PyQt5.QtGui import ( QStandardItem, @@ -43,15 +42,15 @@ from PyQt5.QtWidgets import ( QAbstractScrollArea, QStyledItemDelegate, ) -from fuzzywuzzy import process from ..log import get_logger from ._style import ( _font, DpiAwareFont, - hcolor, + # hcolor, ) +from ..data import feed log = get_logger(__name__) @@ -90,10 +89,34 @@ class CompleterView(QTreeView): def __init__( self, parent=None, + labels: List[str] = [], ) -> None: super().__init__(parent) + model = QStandardItemModel(self) + self.labels = labels + + # a std "tabular" config + self.setItemDelegate(SimpleDelegate()) + self.setModel(model) + self.setAlternatingRowColors(True) + self.setIndentation(1) + + # self.setUniformRowHeights(True) + # self.setColumnWidth(0, 3) + + # ux settings + self.setItemsExpandable(True) + self.setExpandsOnDoubleClick(False) + self.setAnimated(False) + self.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) + + self.setSizeAdjustPolicy(QAbstractScrollArea.AdjustIgnored) + + # column headers + model.setHorizontalHeaderLabels(labels) + self._font_size: int = 0 # pixels # self._cache: Dict[str, List[str]] = {} @@ -115,23 +138,6 @@ class CompleterView(QTreeView): # # psh.setHeight(12) # return QSize(-1, height) - def resize(self): - model = self.model() - cols = model.columnCount() - - for i in range(cols): - self.resizeColumnToContents(i) - - # inclusive of search bar and header "rows" in pixel terms - rows = model.rowCount() + 2 - # max_rows = 8 # 6 + search and headers - row_px = self.rowHeight(self.currentIndex()) - # print(f'font_h: {font_h}\n px_height: {px_height}') - - # TODO: probably make this more general / less hacky - self.setMinimumSize(self.width(), rows * row_px) - self.setMaximumSize(self.width(), rows * row_px) - def set_font_size(self, size: int = 18): # dpi_px_size = _font.px_size print(size) @@ -148,6 +154,19 @@ class CompleterView(QTreeView): ) -> None: model = self.model() + model.clear() + model.setHorizontalHeaderLabels(self.labels) + + # TODO: wtf.. this model shit + # row_count = model.rowCount() + # if row_count > 0: + # model.removeRows( + # 0, + # row_count, + + # # root index + # model.index(0, 0, QModelIndex()), + # ) for i, s in enumerate(results): @@ -160,49 +179,40 @@ class CompleterView(QTreeView): # Add the item to the model model.appendRow([src, ix, item]) - def find_matches( - self, - field: str, - txt: str, - ) -> List[QStandardItem]: + def show_matches(self) -> None: + # print(f"SHOWING {self}") + self.show() + self.resize() + + def resize(self): model = self.model() - items = model.findItems( - txt, - Qt.MatchContains, - self.field_to_col(field), - ) + cols = model.columnCount() + for i in range(cols): + self.resizeColumnToContents(i) -def mk_completer_view( + # inclusive of search bar and header "rows" in pixel terms + rows = model.rowCount() + 2 + print(f'row count: {rows}') + # max_rows = 8 # 6 + search and headers + row_px = self.rowHeight(self.currentIndex()) + # print(f'font_h: {font_h}\n px_height: {px_height}') - labels: List[str], + # TODO: probably make this more general / less hacky + self.setMinimumSize(self.width(), rows * row_px) + self.setMaximumSize(self.width(), rows * row_px) -) -> QTreeView: - - tree = CompleterView() - model = QStandardItemModel(tree) - - # a std "tabular" config - tree.setItemDelegate(SimpleDelegate()) - tree.setModel(model) - tree.setAlternatingRowColors(True) - tree.setIndentation(1) - - # tree.setUniformRowHeights(True) - # tree.setColumnWidth(0, 3) - - # ux settings - tree.setItemsExpandable(True) - tree.setExpandsOnDoubleClick(False) - tree.setAnimated(False) - tree.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) - - tree.setSizeAdjustPolicy(QAbstractScrollArea.AdjustIgnored) - - # column headers - model.setHorizontalHeaderLabels(labels) - - return tree + # def find_matches( + # self, + # field: str, + # txt: str, + # ) -> List[QStandardItem]: + # model = self.model() + # items = model.findItems( + # txt, + # Qt.MatchContains, + # self.field_to_col(field), + # ) class FontSizedQLineEdit(QtWidgets.QLineEdit): @@ -246,7 +256,7 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): def show(self) -> None: super().show() - self.show_matches() + self.view.show_matches() # self.view.show() # self.view.resize() @@ -254,7 +264,7 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): def view(self) -> CompleterView: if self._view is None: - view = mk_completer_view(['src', 'i', 'symbol']) + view = CompleterView(labels=['src', 'i', 'symbol']) # print('yo') # self.chart_app.vbox.addWidget(view) @@ -264,13 +274,6 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): return self._view - def show_matches(self): - view = self.view - view.set_font_size(self.dpi_font.px_size) - view.show() - # scale columns - view.resize() - def sizeHint(self) -> QtCore.QSize: """ Scale edit box to size of dpi aware font. @@ -307,102 +310,163 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): # # ev.accept() -async def handle_keyboard_input( - self, +async def fill_results( + search: FontSizedQLineEdit, + symsearch: Callable[..., Awaitable], recv_chan: trio.abc.ReceiveChannel, + # pattern: str, ) -> None: - async for key, mods, txt in recv_chan: + sel = search.view.selectionModel() + model = search.view.model() - # by default we don't mart it as consumed? - # ev.ignore() + async for pattern in recv_chan: + # so symbol search + # pattern = search.text() + # print(f'searching for: {pattern}') - print(f'key: {key}, mods: {mods}, txt: {txt}') + results = await symsearch(pattern) + # print(f'results\n:{pformat(results)}') - if key in (Qt.Key_Enter, Qt.Key_Return): + if results: + # print(f"results: {results}") - print(f'Requesting symbol: {self.text()}') - - # TODO: ensure there is a matching completion or error and - # do nothing - symbol = self.text() - - app = self.chart_app - self.chart_app.load_symbol( - app.linkedcharts.symbol.brokers[0], - symbol, - 'info', + # TODO: indented branch results for each provider + search.view.set_results( + [item['altname'] for item in + results['kraken'].values()] ) - # release kb control of search bar - self.unfocus() - continue - # return - - ctrl = False - if mods == Qt.ControlModifier: - ctrl = True - - view = self.view - model = view.model() - nidx = cidx = view.currentIndex() - sel = view.selectionModel() - # sel.clear() - - # selection tips: - # - get parent: self.index(row, 0) - # - first item index: index = self.index(0, 0, parent) - - if ctrl: - # we're in select mode or cancelling - - if key == Qt.Key_C: - self.unfocus() - - # kill the search and focus back on main chart - if self.chart_app: - self.chart_app.linkedcharts.focus() - - # return - continue - - # result selection nav - if key in (Qt.Key_K, Qt.Key_J): - - if key == Qt.Key_K: - # self.view.setFocus() - nidx = view.indexAbove(cidx) - print('move up') - - elif key == Qt.Key_J: - # self.view.setFocus() - nidx = view.indexBelow(cidx) - print('move down') - - # select row without selecting.. :eye_rollzz: - # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex - if nidx.isValid(): - sel.setCurrentIndex( - nidx, - QItemSelectionModel.ClearAndSelect | - QItemSelectionModel.Rows - ) - - # TODO: make this not hard coded to 2 - # and use the ``CompleterView`` schema/settings - # to figure out the desired field(s) - value = model.item(nidx.row(), 2).text() - print(f'value: {value}') - self.setText(value) - - else: + # XXX: these 2 lines MUST be in sequence !? sel.setCurrentIndex( model.index(0, 0, QModelIndex()), QItemSelectionModel.ClearAndSelect | # type: ignore[arg-type] QItemSelectionModel.Rows ) + search.show() - self.show_matches() + +async def handle_keyboard_input( + search: FontSizedQLineEdit, + recv_chan: trio.abc.ReceiveChannel, +) -> None: + + # startup + view = search.view + view.set_font_size(search.dpi_font.px_size) + model = view.model() + nidx = cidx = view.currentIndex() + sel = view.selectionModel() + # sel.clear() + + symsearch = feed.get_multi_search() + + send, recv = trio.open_memory_channel(16) + + async with trio.open_nursery() as n: + # TODO: async debouncing! + n.start_soon( + fill_results, + search, + symsearch, + recv, + # pattern, + ) + + async for key, mods, txt in recv_chan: + + # startup + # view = search.view + # view.set_font_size(search.dpi_font.px_size) + # model = view.model() + nidx = cidx = view.currentIndex() + # sel = view.selectionModel() + + # by default we don't mart it as consumed? + # ev.ignore() + search.show() + + log.debug(f'key: {key}, mods: {mods}, txt: {txt}') + + ctrl = False + if mods == Qt.ControlModifier: + ctrl = True + + if key in (Qt.Key_Enter, Qt.Key_Return): + + value = model.item(nidx.row(), 2).text() + + log.info(f'Requesting symbol: {value}') + + app = search.chart_app + search.chart_app.load_symbol( + app.linkedcharts.symbol.brokers[0], + value, + 'info', + ) + + # release kb control of search bar + search.unfocus() + continue + + # selection tips: + # - get parent: search.index(row, 0) + # - first item index: index = search.index(0, 0, parent) + + if ctrl: + # we're in select mode or cancelling + + if key == Qt.Key_C: + search.unfocus() + + # kill the search and focus back on main chart + if search.chart_app: + search.chart_app.linkedcharts.focus() + + continue + + # result selection nav + if key in (Qt.Key_K, Qt.Key_J): + + if key == Qt.Key_K: + # search.view.setFocus() + nidx = view.indexAbove(cidx) + print('move up') + + elif key == Qt.Key_J: + # search.view.setFocus() + nidx = view.indexBelow(cidx) + print('move down') + + # select row without selecting.. :eye_rollzz: + # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex + if nidx.isValid(): + sel.setCurrentIndex( + nidx, + QItemSelectionModel.ClearAndSelect | + QItemSelectionModel.Rows + ) + + # TODO: make this not hard coded to 2 + # and use the ``CompleterView`` schema/settings + # to figure out the desired field(s) + value = model.item(nidx.row(), 2).text() + print(f'value: {value}') + # search.setText(value) + # continue + + else: + # auto-select the top matching result + sel.setCurrentIndex( + model.index(0, 0, QModelIndex()), + QItemSelectionModel.ClearAndSelect | + QItemSelectionModel.Rows + ) + search.view.show_matches() + + else: + # relay to completer task + send.send_nowait(search.text()) if __name__ == '__main__': @@ -413,13 +477,14 @@ if __name__ == '__main__': syms = [ 'XMRUSD', 'XBTUSD', + 'ETHUSD', 'XMRXBT', 'XDGUSD', 'ADAUSD', ] # results.setFocusPolicy(Qt.NoFocus) - view = mk_completer_view(['src', 'i', 'symbol']) + view = CompleterView(['src', 'i', 'symbol']) search = FontSizedQLineEdit(None, view=view) search.view.set_results(syms) @@ -434,7 +499,6 @@ if __name__ == '__main__': main = W() main.vbox.addWidget(search) main.vbox.addWidget(view) - # main.show() search.show() sys.exit(app.exec_()) From c9efbcc6d2397a393a1a5f0f38bc0886240b50dc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 6 May 2021 16:41:57 -0400 Subject: [PATCH 19/81] Drop completion list from startup --- piker/ui/_chart.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 0057ac10..01d3af35 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1600,16 +1600,6 @@ async def _async_main( chart_app.vbox.addWidget(search.view) chart_app.search = search - search.view.set_results([ - 'ETHUSD', - 'XMRUSD', - 'XBTUSD', - 'XMRXBT', - # 'XMRXBT', - # 'XDGUSD', - # 'ADAUSD', - ]) - # this internally starts a ``chart_symbol()`` task above chart_app.load_symbol(brokername, sym, loglevel) From 478b1147084809c58c1131d40fa732da36ed9226 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 10 May 2021 10:10:53 -0400 Subject: [PATCH 20/81] First draft completion in background task --- piker/ui/_search.py | 143 +++++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 75 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index f8681f76..658ac23a 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -18,8 +18,12 @@ qompleterz: embeddable search and complete using trio, Qt and fuzzywuzzy. """ -from typing import List, Optional, Callable, Awaitable import sys +import time +from typing import ( + List, Optional, Callable, + Awaitable, Sequence, Dict, +) # from pprint import pformat from PyQt5 import QtCore, QtGui @@ -150,7 +154,7 @@ class CompleterView(QTreeView): def set_results( self, - results: List[str], + results: Dict[str, Sequence[str]], ) -> None: model = self.model() @@ -167,17 +171,19 @@ class CompleterView(QTreeView): # # root index # model.index(0, 0, QModelIndex()), # ) + for key, values in results.items(): - for i, s in enumerate(results): + # values just needs to be sequence-like + for i, s in enumerate(values): - ix = QStandardItem(str(i)) - item = QStandardItem(s) - # item.setCheckable(False) + ix = QStandardItem(str(i)) + item = QStandardItem(s) + # item.setCheckable(False) - src = QStandardItem('kraken') + src = QStandardItem(key) - # Add the item to the model - model.appendRow([src, ix, item]) + # Add the item to the model + model.appendRow([src, ix, item]) def show_matches(self) -> None: # print(f"SHOWING {self}") @@ -291,23 +297,9 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): if self.view: self.view.hide() - # def keyPressEvent(self, ev: QEvent) -> None: - # # XXX: we unpack here because apparently doing it - # # after pop from the mem chan isn't showing the same - # # event object? no clue wtf is going on there, likely - # # something to do with Qt internals and calling the - # # parent handler? - # key = ev.key() - # mods = ev.modifiers() - # txt = self.text() - - # # run async processing - # self._send_chan.send_nowait((key, mods, txt)) - - # super().keyPressEvent(ev) - - # # ev.accept() +_ongoing_search: trio.CancelScope = None +_search_enabled: bool = False async def fill_results( @@ -316,28 +308,37 @@ async def fill_results( recv_chan: trio.abc.ReceiveChannel, # pattern: str, ) -> None: + """Task to search through providers and fill in possible + completion results. + """ + global _ongoing_search, _search_enabled + + view = search.view sel = search.view.selectionModel() model = search.view.model() async for pattern in recv_chan: - # so symbol search - # pattern = search.text() - # print(f'searching for: {pattern}') - results = await symsearch(pattern) - # print(f'results\n:{pformat(results)}') + if not _search_enabled: + log.debug(f'Ignoring search for {pattern}') + continue - if results: - # print(f"results: {results}") + with trio.CancelScope() as cs: + _ongoing_search = cs + results = await symsearch(pattern) + _ongoing_search = None + + if results and not cs.cancelled_caught: # TODO: indented branch results for each provider - search.view.set_results( - [item['altname'] for item in - results['kraken'].values()] - ) + view.set_results(results) + # [item['altname'] for item in + # results['kraken'].values()] + # ) - # XXX: these 2 lines MUST be in sequence !? + # XXX: these 2 lines MUST be in sequence in order + # to get the view to show right after typing input. sel.setCurrentIndex( model.index(0, 0, QModelIndex()), QItemSelectionModel.ClearAndSelect | # type: ignore[arg-type] @@ -347,46 +348,43 @@ async def fill_results( async def handle_keyboard_input( + search: FontSizedQLineEdit, recv_chan: trio.abc.ReceiveChannel, + ) -> None: + global _ongoing_search, _search_enabled + # startup view = search.view view.set_font_size(search.dpi_font.px_size) model = view.model() nidx = cidx = view.currentIndex() sel = view.selectionModel() - # sel.clear() symsearch = feed.get_multi_search() - send, recv = trio.open_memory_channel(16) async with trio.open_nursery() as n: - # TODO: async debouncing! + # TODO: async debouncing? n.start_soon( fill_results, search, symsearch, recv, - # pattern, ) + last_time = time.time() + async for key, mods, txt in recv_chan: - - # startup - # view = search.view - # view.set_font_size(search.dpi_font.px_size) - # model = view.model() - nidx = cidx = view.currentIndex() - # sel = view.selectionModel() - - # by default we don't mart it as consumed? - # ev.ignore() - search.show() + # TODO: move this logic into completer task + now = time.time() + period = now - last_time + last_time = now log.debug(f'key: {key}, mods: {mods}, txt: {txt}') + nidx = cidx = view.currentIndex() ctrl = False if mods == Qt.ControlModifier: @@ -394,7 +392,15 @@ async def handle_keyboard_input( if key in (Qt.Key_Enter, Qt.Key_Return): - value = model.item(nidx.row(), 2).text() + if _ongoing_search is not None: + _ongoing_search.cancel() + _search_enabled = False + + node = model.item(nidx.row(), 2) + if node: + value = node.text() + else: + continue log.info(f'Requesting symbol: {value}') @@ -410,12 +416,14 @@ async def handle_keyboard_input( continue # selection tips: - # - get parent: search.index(row, 0) - # - first item index: index = search.index(0, 0, parent) + # - get parent node: search.index(row, 0) + # - first node index: index = search.index(0, 0, parent) + # - root node index: index = search.index(0, 0, QModelIndex()) + # we're in select mode or cancelling if ctrl: - # we're in select mode or cancelling + # cancel and close if key == Qt.Key_C: search.unfocus() @@ -429,14 +437,10 @@ async def handle_keyboard_input( if key in (Qt.Key_K, Qt.Key_J): if key == Qt.Key_K: - # search.view.setFocus() nidx = view.indexAbove(cidx) - print('move up') elif key == Qt.Key_J: - # search.view.setFocus() nidx = view.indexBelow(cidx) - print('move down') # select row without selecting.. :eye_rollzz: # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex @@ -451,22 +455,11 @@ async def handle_keyboard_input( # and use the ``CompleterView`` schema/settings # to figure out the desired field(s) value = model.item(nidx.row(), 2).text() - print(f'value: {value}') - # search.setText(value) - # continue - - else: - # auto-select the top matching result - sel.setCurrentIndex( - model.index(0, 0, QModelIndex()), - QItemSelectionModel.ClearAndSelect | - QItemSelectionModel.Rows - ) - search.view.show_matches() - else: - # relay to completer task - send.send_nowait(search.text()) + if period >= 0.1: + _search_enabled = True + # relay to completer task + send.send_nowait(search.text()) if __name__ == '__main__': From 5766dd518d47b3d756bc01c66e5b80cc572ddcbf Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 10 May 2021 10:17:06 -0400 Subject: [PATCH 21/81] Enforce lower case symbols across providers --- piker/data/feed.py | 31 +++++++++++++++++++------------ piker/ui/_chart.py | 4 ++++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index b9df8595..cdd19070 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -379,13 +379,25 @@ class Feed: @asynccontextmanager async def open_symbol_search(self) -> AsyncIterator[dict]: + open_search = getattr(self.mod, 'open_symbol_search', None) + if open_search is None: + + # just return a pure pass through searcher + async def passthru(text: str) -> Dict[str, Any]: + return text + + self.search = passthru + yield self.search + self.search = None + return + async with self._brokerd_portal.open_context( - - self.mod.open_symbol_search, - + open_search, ) as (ctx, cache): - async with ctx.open_stream() as stream: + # shield here since we expect the search rpc to be + # cancellable by the user as they see fit. + async with ctx.open_stream(shield=True) as stream: async def search(text: str) -> Dict[str, Any]: await stream.send(text) @@ -448,7 +460,7 @@ async def open_feed( """ global _cache, _cache_lock - sym = symbols[0] + sym = symbols[0].lower() # TODO: feed cache locking, right now this is causing # issues when reconncting to a long running emsd? @@ -526,11 +538,6 @@ async def open_feed( _cache[(brokername, sym)] = feed - try: - async with feed.open_symbol_search(): - yield feed + async with feed.open_symbol_search(): + yield feed - finally: - # always cancel the far end producer task - with trio.CancelScope(shield=True): - await stream.aclose() diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 01d3af35..7b574422 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -94,6 +94,7 @@ class ChartSpace(QtGui.QWidget): # self.init_strategy_ui() self.vbox.addLayout(self.toolbar_layout) self.vbox.addLayout(self.hbox) + self._chart_cache = {} self.linkedcharts: 'LinkedSplitCharts' = None self.symbol_label: Optional[QtGui.QLabel] = None @@ -135,6 +136,9 @@ class ChartSpace(QtGui.QWidget): Expects a ``numpy`` structured array containing all the ohlcv fields. """ + # our symbol key style is always lower case + symbol_key = symbol_key.lower() + linkedcharts = self._chart_cache.get(symbol_key) if not self.vbox.isEmpty(): From 82a8e0a7b6cec0f55d794c0db046375441aaecb8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 10 May 2021 10:17:34 -0400 Subject: [PATCH 22/81] Accept lower case sym requests in kraken backend --- piker/brokers/kraken.py | 64 ++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index 0dbab324..09a92c90 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -289,32 +289,6 @@ async def get_client() -> Client: yield client -@tractor.context -async def open_symbol_search( - ctx: tractor.Context, -) -> Client: - async with open_cached_client('kraken') as client: - - # load all symbols locally for fast search - cache = await client.cache_symbols() - await ctx.started(cache) - - async with ctx.open_stream() as stream: - - async for pattern in stream: - - matches = fuzzy.extractBests( - pattern, - cache, - score_cutoff=50, - ) - # repack in dict form - await stream.send( - {item[0]['altname']: item[0] - for item in matches} - ) - - async def stream_messages(ws): too_slow_count = last_hb = 0 @@ -397,7 +371,8 @@ def normalize( # seriously eh? what's with this non-symmetry everywhere # in subscription systems... - topic = quote['pair'].replace('/', '') + # XXX: piker style is always lowercases symbols. + topic = quote['pair'].replace('/', '').lower() # print(quote) return topic, quote @@ -558,6 +533,10 @@ async def stream_quotes( # keep client cached for real-time section for sym in symbols: + + # transform to upper since piker style is always lower + sym = sym.upper() + si = Pair(**await client.symbol_info(sym)) # validation syminfo = si.dict() syminfo['price_tick_size'] = 1 / 10**si.pair_decimals @@ -565,7 +544,7 @@ async def stream_quotes( sym_infos[sym] = syminfo ws_pairs[sym] = si.wsname - symbol = symbols[0] + symbol = symbols[0].lower() init_msgs = { # pass back token, and bool, signalling if we're the writer @@ -653,8 +632,35 @@ async def stream_quotes( elif typ == 'l1': quote = ohlc - topic = quote['symbol'] + topic = quote['symbol'].lower() # XXX: format required by ``tractor.msg.pub`` # requires a ``Dict[topic: str, quote: dict]`` await send_chan.send({topic: quote}) + + + +@tractor.context +async def open_symbol_search( + ctx: tractor.Context, +) -> Client: + async with open_cached_client('kraken') as client: + + # load all symbols locally for fast search + cache = await client.cache_symbols() + await ctx.started(cache) + + async with ctx.open_stream() as stream: + + async for pattern in stream: + + matches = fuzzy.extractBests( + pattern, + cache, + score_cutoff=50, + ) + # repack in dict form + await stream.send( + {item[0]['altname']: item[0] + for item in matches} + ) From ef1b0911f3f705a08a962ea1feffb7e032782bd7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 10 May 2021 10:23:08 -0400 Subject: [PATCH 23/81] Add symbol search to ib --- piker/brokers/ib.py | 139 +++++++++++++++++++++++++++++--------------- 1 file changed, 91 insertions(+), 48 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 112e935e..98c3a979 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -45,7 +45,9 @@ from ib_insync.objects import Position import ib_insync as ibis from ib_insync.wrapper import Wrapper from ib_insync.client import Client as ib_Client +from fuzzywuzzy import process as fuzzy +from .api import open_cached_client from ..log import get_logger, get_console_log from .._daemon import maybe_spawn_brokerd from ..data._source import from_df @@ -322,7 +324,7 @@ class Client: sym, exch = symbol.upper().rsplit('.', maxsplit=1) except ValueError: # likely there's an embedded `.` for a forex pair - await tractor.breakpoint() + breakpoint() # futes if exch in ('GLOBEX', 'NYMEX', 'CME', 'CMECRYPTO'): @@ -350,10 +352,13 @@ class Client: if exch in ('PURE', 'TSE'): # non-yankee currency = 'CAD' - if exch in ('PURE', 'TSE'): - # stupid ib... - primaryExchange = exch - exch = 'SMART' + # stupid ib... + primaryExchange = exch + exch = 'SMART' + + else: + exch = 'SMART' + primaryExchange = exch con = ibis.Stock( symbol=sym, @@ -994,23 +999,31 @@ async def stream_quotes( } } + con = first_ticker.contract + + # should be real volume for this contract by default + calc_price = False + # check for special contract types if type(first_ticker.contract) not in (ibis.Commodity, ibis.Forex): - suffix = 'exchange' - # should be real volume for this contract - calc_price = False + + suffix = con.primaryExchange + if not suffix: + suffix = con.exchange + else: # commodities and forex don't have an exchange name and # no real volume so we have to calculate the price - suffix = 'secType' + suffix = con.secType + # no real volume on this tract calc_price = True - # pass first quote asap quote = normalize(first_ticker, calc_price=calc_price) con = quote['contract'] - topic = '.'.join((con['symbol'], con[suffix])).lower() + topic = '.'.join((con['symbol'], suffix)).lower() quote['symbol'] = topic + # pass first quote asap first_quote = {topic: quote} # ugh, clear ticks since we've consumed them @@ -1022,50 +1035,50 @@ async def stream_quotes( task_status.started((init_msgs, first_quote)) if type(first_ticker.contract) not in (ibis.Commodity, ibis.Forex): - suffix = 'exchange' - calc_price = False # should be real volume for contract + # suffix = 'exchange' + # calc_price = False # should be real volume for contract # wait for real volume on feed (trading might be closed) - async with aclosing(stream): + while True: - async for ticker in stream: - - # for a real volume contract we rait for the first - # "real" trade to take place - if not calc_price and not ticker.rtTime: - # spin consuming tickers until we get a real market datum - log.debug(f"New unsent ticker: {ticker}") - continue - else: - log.debug("Received first real volume tick") - # ugh, clear ticks since we've consumed them - # (ahem, ib_insync is truly stateful trash) - ticker.ticks = [] - - # XXX: this works because we don't use - # ``aclosing()`` above? - break - - # tell caller quotes are now coming in live - feed_is_live.set() - - async for ticker in stream: - - # print(ticker.vwap) - quote = normalize( - ticker, - calc_price=calc_price - ) - - con = quote['contract'] - topic = '.'.join((con['symbol'], con[suffix])).lower() - quote['symbol'] = topic - - await send_chan.send({topic: quote}) + ticker = await stream.receive() + # for a real volume contract we rait for the first + # "real" trade to take place + if not calc_price and not ticker.rtTime: + # spin consuming tickers until we get a real market datum + log.debug(f"New unsent ticker: {ticker}") + continue + else: + log.debug("Received first real volume tick") # ugh, clear ticks since we've consumed them + # (ahem, ib_insync is truly stateful trash) ticker.ticks = [] + # XXX: this works because we don't use + # ``aclosing()`` above? + break + + # tell caller quotes are now coming in live + feed_is_live.set() + + async with aclosing(stream): + async for ticker in stream: + + # print(ticker.vwap) + quote = normalize( + ticker, + calc_price=calc_price + ) + + # con = quote['contract'] + # topic = '.'.join((con['symbol'], suffix)).lower() + quote['symbol'] = topic + await send_chan.send({topic: quote}) + + # ugh, clear ticks since we've consumed them + ticker.ticks = [] + def pack_position(pos: Position) -> Dict[str, Any]: con = pos.contract @@ -1183,3 +1196,33 @@ async def stream_trades( continue yield {'local_trades': (event_name, msg)} + + +@tractor.context +async def open_symbol_search( + ctx: tractor.Context, +) -> Client: + async with open_cached_client('ib') as client: + + # load all symbols locally for fast search + await ctx.started({}) + + async with ctx.open_stream() as stream: + + async for pattern in stream: + + if not pattern: + # will get error on empty request + continue + + results = await client.search_stocks(pattern=pattern, upto=5) + + matches = fuzzy.extractBests( + pattern, + results, + score_cutoff=50, + ) + await stream.send( + {item[2]: item[0] + for item in matches} + ) From 25d7122cb601236f346e5733d516d4f353fdee5a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 11 May 2021 17:26:27 -0400 Subject: [PATCH 24/81] Throttle requests using a static "typing paused period" --- piker/ui/_search.py | 90 +++++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 658ac23a..0feeb4a8 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -19,7 +19,6 @@ qompleterz: embeddable search and complete using trio, Qt and fuzzywuzzy. """ import sys -import time from typing import ( List, Optional, Callable, Awaitable, Sequence, Dict, @@ -298,50 +297,86 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): self.view.hide() -_ongoing_search: trio.CancelScope = None +_search_active: trio.Event = trio.Event() _search_enabled: bool = False async def fill_results( + search: FontSizedQLineEdit, symsearch: Callable[..., Awaitable], recv_chan: trio.abc.ReceiveChannel, - # pattern: str, + pause_time: float = 0.25, + ) -> None: """Task to search through providers and fill in possible completion results. """ - global _ongoing_search, _search_enabled + global _search_active, _search_enabled view = search.view sel = search.view.selectionModel() model = search.view.model() - async for pattern in recv_chan: + last_search_text = '' + last_text = search.text() + repeats = 0 - if not _search_enabled: - log.debug(f'Ignoring search for {pattern}') + while True: + + last_text = search.text() + await _search_active.wait() + + with trio.move_on_after(pause_time) as cs: + # cs.shield = True + pattern = await recv_chan.receive() + print(pattern) + + # during fast multiple key inputs, wait until a pause + # (in typing) to initiate search + if not cs.cancelled_caught: + log.debug(f'Ignoring fast input for {pattern}') continue - with trio.CancelScope() as cs: - _ongoing_search = cs - results = await symsearch(pattern) - _ongoing_search = None + text = search.text() + print(f'search: {text}') - if results and not cs.cancelled_caught: + if not text: + print('idling') + _search_active = trio.Event() + continue + + if text == last_text: + repeats += 1 + + if repeats > 1: + _search_active = trio.Event() + repeats = 0 + + if not _search_enabled: + print('search not ENABLED?') + continue + + if last_search_text and last_search_text == text: + continue + + log.debug(f'Search req for {text}') + + last_search_text = text + results = await symsearch(text) + log.debug(f'Received search result {results}') + + if results and _search_enabled: # TODO: indented branch results for each provider view.set_results(results) - # [item['altname'] for item in - # results['kraken'].values()] - # ) # XXX: these 2 lines MUST be in sequence in order # to get the view to show right after typing input. sel.setCurrentIndex( model.index(0, 0, QModelIndex()), - QItemSelectionModel.ClearAndSelect | # type: ignore[arg-type] + QItemSelectionModel.ClearAndSelect | QItemSelectionModel.Rows ) search.show() @@ -354,7 +389,7 @@ async def handle_keyboard_input( ) -> None: - global _ongoing_search, _search_enabled + global _search_active, _search_enabled # startup view = search.view @@ -375,13 +410,7 @@ async def handle_keyboard_input( recv, ) - last_time = time.time() - async for key, mods, txt in recv_chan: - # TODO: move this logic into completer task - now = time.time() - period = now - last_time - last_time = now log.debug(f'key: {key}, mods: {mods}, txt: {txt}') nidx = cidx = view.currentIndex() @@ -392,10 +421,6 @@ async def handle_keyboard_input( if key in (Qt.Key_Enter, Qt.Key_Return): - if _ongoing_search is not None: - _ongoing_search.cancel() - _search_enabled = False - node = model.item(nidx.row(), 2) if node: value = node.text() @@ -411,6 +436,7 @@ async def handle_keyboard_input( 'info', ) + _search_enabled = False # release kb control of search bar search.unfocus() continue @@ -422,7 +448,6 @@ async def handle_keyboard_input( # we're in select mode or cancelling if ctrl: - # cancel and close if key == Qt.Key_C: search.unfocus() @@ -435,6 +460,7 @@ async def handle_keyboard_input( # result selection nav if key in (Qt.Key_K, Qt.Key_J): + _search_enabled = False if key == Qt.Key_K: nidx = view.indexAbove(cidx) @@ -456,10 +482,10 @@ async def handle_keyboard_input( # to figure out the desired field(s) value = model.item(nidx.row(), 2).text() else: - if period >= 0.1: - _search_enabled = True - # relay to completer task - send.send_nowait(search.text()) + # relay to completer task + _search_enabled = True + send.send_nowait(search.text()) + _search_active.set() if __name__ == '__main__': From e5e9a7c582582562d4cb56530bb6e18bc7cf3b9b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 12 May 2021 08:32:15 -0400 Subject: [PATCH 25/81] Add symbol searching for ib backend Obviously this only supports stocks to start, it looks like we might actually have to hard code some of the futures/forex/cmdtys that don't have a search.. so lame. Special throttling is added here since the api will grog out at anything more then 1Hz. Additionally, decouple the bar loading request error handling from the shm pushing loop so that we can always recover from a historical bars throttle-error even if it's on the first try for a new symbol. --- piker/brokers/ib.py | 183 +++++++++++++++++++++++++++++--------------- 1 file changed, 120 insertions(+), 63 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 98c3a979..eea26b62 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -46,8 +46,8 @@ import ib_insync as ibis from ib_insync.wrapper import Wrapper from ib_insync.client import Client as ib_Client from fuzzywuzzy import process as fuzzy +import numpy as np -from .api import open_cached_client from ..log import get_logger, get_console_log from .._daemon import maybe_spawn_brokerd from ..data._source import from_df @@ -143,11 +143,21 @@ class NonShittyIB(ibis.IB): # map of symbols to contract ids _adhoc_cmdty_data_map = { # https://misc.interactivebrokers.com/cstools/contract_info/v3.10/index.php?action=Conid%20Info&wlId=IB&conid=69067924 - # NOTE: cmdtys don't have trade data: + + # NOTE: some cmdtys/metals don't have trade data like gold/usd: # https://groups.io/g/twsapi/message/44174 'XAUUSD': ({'conId': 69067924}, {'whatToShow': 'MIDPOINT'}), } +_adhoc_futes_set = { + 'nq.globex', + 'mnq.globex', + 'es.globex', + 'mes.globex', +} + + # https://misc.interactivebrokers.com/cstools/contract_info/v3.10/index.php?action=Conid%20Info&wlId=IB&conid=69067924 + _enters = 0 @@ -650,6 +660,8 @@ async def _aio_run_client_method( if to_trio and 'to_trio' in args: kwargs['to_trio'] = to_trio + log.runtime(f'Running {meth}({kwargs})') + return await async_meth(**kwargs) @@ -786,13 +798,64 @@ def normalize( return data +async def get_bars( + sym: str, + end_dt: str = "", +) -> (dict, np.ndarray): + + _err = None + + for _ in range(1): + try: + + bars, bars_array = await _trio_run_client_method( + method='bars', + symbol=sym, + end_dt=end_dt, + ) + + if bars_array is None: + raise SymbolNotFound(sym) + + next_dt = bars[0].date + + return bars, bars_array, next_dt + + except RequestError as err: + _err = err + + # TODO: retreive underlying ``ib_insync`` error? + if err.code == 162: + + if 'HMDS query returned no data' in err.message: + # means we hit some kind of historical "dead zone" + # and further requests seem to always cause + # throttling despite the rps being low + break + + else: + log.exception( + "Data query rate reached: Press `ctrl-alt-f`" + "in TWS" + ) + + # TODO: should probably create some alert on screen + # and then somehow get that to trigger an event here + # that restarts/resumes this task? + await tractor.breakpoint() + + else: # throttle wasn't fixed so error out immediately + raise _err + + async def backfill_bars( sym: str, shm: ShmArray, # type: ignore # noqa # count: int = 20, # NOTE: any more and we'll overrun underlying buffer - count: int = 10, # NOTE: any more and we'll overrun the underlying buffer + count: int = 6, # NOTE: any more and we'll overrun the underlying buffer task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED, + ) -> None: """Fill historical bars into shared mem / storage afap. @@ -800,10 +863,7 @@ async def backfill_bars( https://github.com/pikers/piker/issues/128 """ - first_bars, bars_array = await _trio_run_client_method( - method='bars', - symbol=sym, - ) + first_bars, bars_array, next_dt = await get_bars(sym) # write historical data to buffer shm.push(bars_array) @@ -812,46 +872,12 @@ async def backfill_bars( task_status.started(cs) - next_dt = first_bars[0].date - i = 0 while i < count: - try: - bars, bars_array = await _trio_run_client_method( - method='bars', - symbol=sym, - end_dt=next_dt, - ) - - if bars_array is None: - raise SymbolNotFound(sym) - - shm.push(bars_array, prepend=True) - i += 1 - next_dt = bars[0].date - - except RequestError as err: - # TODO: retreive underlying ``ib_insync`` error? - - if err.code == 162: - - if 'HMDS query returned no data' in err.message: - # means we hit some kind of historical "dead zone" - # and further requests seem to always cause - # throttling despite the rps being low - break - - else: - log.exception( - "Data query rate reached: Press `ctrl-alt-f`" - "in TWS" - ) - - # TODO: should probably create some alert on screen - # and then somehow get that to trigger an event here - # that restarts/resumes this task? - await tractor.breakpoint() + bars, bars_array, next_dt = await get_bars(sym, end_dt=next_dt) + shm.push(bars_array, prepend=True) + i += 1 asset_type_map = { @@ -1201,28 +1227,59 @@ async def stream_trades( @tractor.context async def open_symbol_search( ctx: tractor.Context, -) -> Client: - async with open_cached_client('ib') as client: +) -> None: + # async with open_cached_client('ib') as client: - # load all symbols locally for fast search - await ctx.started({}) + # load all symbols locally for fast search + await ctx.started({}) - async with ctx.open_stream() as stream: + async with ctx.open_stream() as stream: - async for pattern in stream: + last = time.time() - if not pattern: - # will get error on empty request - continue + async for pattern in stream: + log.debug(f'received {pattern}') + now = time.time() - results = await client.search_stocks(pattern=pattern, upto=5) + assert pattern, 'IB can not accept blank search pattern' - matches = fuzzy.extractBests( - pattern, - results, - score_cutoff=50, - ) - await stream.send( - {item[2]: item[0] - for item in matches} - ) + # throttle search requests to no faster then 1Hz + diff = now - last + if diff < 1.0: + log.debug('throttle sleeping') + await trio.sleep(diff) + try: + pattern = stream.receive_nowait() + # if new: + # pattern = new + except trio.WouldBlock: + pass + + log.debug(f'searching for {pattern}') + # await tractor.breakpoint() + last = time.time() + results = await _trio_run_client_method( + method='search_stocks', + pattern=pattern, + upto=5, + ) + log.debug(f'got results {results.keys()}') + # results = await client.search_stocks( + # pattern=pattern, upto=5) + + # if cs.cancelled_caught: + # print(f'timed out search for {pattern} !?') + # # await tractor.breakpoint() + # await stream.send({}) + # continue + + log.debug("fuzzy matching") + matches = fuzzy.extractBests( + pattern, + results, + score_cutoff=50, + ) + + matches = {item[2]: item[0] for item in matches} + log.debug(f"sending matches: {matches.keys()}") + await stream.send(matches) From f19f4348e0d76288e5bf4d5edf0556e7708fa4bd Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 12 May 2021 08:36:18 -0400 Subject: [PATCH 26/81] Decouple symbol search from feed type --- piker/data/feed.py | 90 ++++++++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index cdd19070..bde8fe72 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -34,7 +34,6 @@ import trio from trio_typing import TaskStatus import tractor from pydantic import BaseModel -from fuzzywuzzy import process as fuzzy from ..brokers import get_brokermod from ..log import get_logger, get_console_log @@ -376,38 +375,6 @@ class Feed: yield self._trade_stream - @asynccontextmanager - async def open_symbol_search(self) -> AsyncIterator[dict]: - - open_search = getattr(self.mod, 'open_symbol_search', None) - if open_search is None: - - # just return a pure pass through searcher - async def passthru(text: str) -> Dict[str, Any]: - return text - - self.search = passthru - yield self.search - self.search = None - return - - async with self._brokerd_portal.open_context( - open_search, - ) as (ctx, cache): - - # shield here since we expect the search rpc to be - # cancellable by the user as they see fit. - async with ctx.open_stream(shield=True) as stream: - - async def search(text: str) -> Dict[str, Any]: - await stream.send(text) - return await stream.receive() - - # deliver search func to consumer - self.search = search - yield search - self.search = None - def sym_to_shm_key( broker: str, @@ -417,7 +384,7 @@ def sym_to_shm_key( # cache of brokernames to feeds -_cache: Dict[str, Feed] = {} +_cache: Dict[str, Callable] = {} _cache_lock: trio.Lock = trio.Lock() @@ -434,21 +401,60 @@ def get_multi_search() -> Callable[..., Awaitable]: async def pack_matches( brokername: str, pattern: str, + search: Callable[..., Awaitable[dict]], ) -> None: - matches[brokername] = await feed.search(pattern) + log.debug(f'Searching {brokername} for "{pattern}"') + matches[brokername] = await search(pattern) # TODO: make this an async stream? async with trio.open_nursery() as n: - for (brokername, startup_sym), feed in _cache.items(): - if feed.search: - n.start_soon(pack_matches, brokername, pattern) + for brokername, search in _cache.items(): + n.start_soon(pack_matches, brokername, pattern, search) return matches return multisearcher +@asynccontextmanager +async def open_symbol_search( + brokermod: ModuleType, + brokerd_portal: tractor._portal.Portal, +) -> AsyncIterator[dict]: + + global _cache + + open_search = getattr(brokermod, 'open_symbol_search', None) + if open_search is None: + + # just return a pure pass through searcher + async def passthru(text: str) -> Dict[str, Any]: + return text + + yield passthru + return + + async with brokerd_portal.open_context( + open_search, + ) as (ctx, cache): + + # shield here since we expect the search rpc to be + # cancellable by the user as they see fit. + async with ctx.open_stream() as stream: + + async def search(text: str) -> Dict[str, Any]: + await stream.send(text) + return await stream.receive() + + # deliver search func to consumer + try: + _cache[brokermod.name] = search + yield search + finally: + _cache.pop(brokermod.name) + + @asynccontextmanager async def open_feed( brokername: str, @@ -536,8 +542,8 @@ async def open_feed( feed._max_sample_rate = max(ohlc_sample_rates) - _cache[(brokername, sym)] = feed - - async with feed.open_symbol_search(): + if brokername in _cache: yield feed - + else: + async with open_symbol_search(mod, feed._brokerd_portal): + yield feed From 2c24c9ef2de117de0fd6ef6140d5aa0c6d5d3756 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 14 May 2021 07:51:42 -0400 Subject: [PATCH 27/81] Compose search bar and view under parent widget --- piker/ui/_search.py | 118 +++++++++++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 46 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 0feeb4a8..3d271589 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -19,6 +19,7 @@ qompleterz: embeddable search and complete using trio, Qt and fuzzywuzzy. """ import sys +from functools import partial from typing import ( List, Optional, Callable, Awaitable, Sequence, Dict, @@ -40,6 +41,7 @@ from PyQt5.QtGui import ( QStandardItemModel, ) from PyQt5.QtWidgets import ( + QWidget, QTreeView, # QListWidgetItem, QAbstractScrollArea, @@ -220,23 +222,21 @@ class CompleterView(QTreeView): # ) -class FontSizedQLineEdit(QtWidgets.QLineEdit): +class SearchBar(QtWidgets.QLineEdit): def __init__( + self, - parent_chart: 'ChartSpace', # noqa + parent: QWidget, + parent_chart: QWidget, # noqa view: Optional[CompleterView] = None, font: DpiAwareFont = _font, + ) -> None: - super().__init__(parent_chart) - # vbox = self.vbox = QtGui.QVBoxLayout(self) - # vbox.addWidget(self) - # self.vbox.setContentsMargins(0, 0, 0, 0) - # self.vbox.setSpacing(2) - - self._view: CompleterView = view + super().__init__(parent) + self.view: CompleterView = view self.dpi_font = font self.chart_app = parent_chart @@ -262,22 +262,6 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): def show(self) -> None: super().show() self.view.show_matches() - # self.view.show() - # self.view.resize() - - @property - def view(self) -> CompleterView: - - if self._view is None: - view = CompleterView(labels=['src', 'i', 'symbol']) - - # print('yo') - # self.chart_app.vbox.addWidget(view) - # self.vbox.addWidget(view) - - self._view = view - - return self._view def sizeHint(self) -> QtCore.QSize: """ @@ -286,7 +270,7 @@ class FontSizedQLineEdit(QtWidgets.QLineEdit): """ psh = super().sizeHint() psh.setHeight(self.dpi_font.px_size + 2) - # psh.setHeight(12) + psh.setWidth(6*6*6) return psh def unfocus(self) -> None: @@ -303,10 +287,10 @@ _search_enabled: bool = False async def fill_results( - search: FontSizedQLineEdit, + search: SearchBar, symsearch: Callable[..., Awaitable], recv_chan: trio.abc.ReceiveChannel, - pause_time: float = 0.25, + pause_time: float = 0.0616, ) -> None: """Task to search through providers and fill in possible @@ -315,17 +299,18 @@ async def fill_results( """ global _search_active, _search_enabled - view = search.view - sel = search.view.selectionModel() - model = search.view.model() + bar = search.bar + view = bar.view + sel = bar.view.selectionModel() + model = bar.view.model() last_search_text = '' - last_text = search.text() + last_text = bar.text() repeats = 0 while True: - last_text = search.text() + last_text = bar.text() await _search_active.wait() with trio.move_on_after(pause_time) as cs: @@ -339,7 +324,7 @@ async def fill_results( log.debug(f'Ignoring fast input for {pattern}') continue - text = search.text() + text = bar.text() print(f'search: {text}') if not text: @@ -379,21 +364,59 @@ async def fill_results( QItemSelectionModel.ClearAndSelect | QItemSelectionModel.Rows ) - search.show() + bar.show() + + +class SearchWidget(QtGui.QWidget): + def __init__( + self, + chart_space: 'ChartSpace', # type: ignore # noqa + columns: List[str] = ['src', 'i', 'symbol'], + parent=None, + ): + super().__init__(parent or chart_space) + + # size it as we specify + self.setSizePolicy( + QtWidgets.QSizePolicy.Fixed, + QtWidgets.QSizePolicy.Fixed, + ) + + self.chart_app = chart_space + self.vbox = QtGui.QVBoxLayout(self) + self.vbox.setContentsMargins(0, 0, 0, 0) + self.vbox.setSpacing(2) + + self.view = CompleterView( + parent=self, + labels=columns, + ) + self.bar = SearchBar( + parent=self, + parent_chart=chart_space, + view=self.view, + ) + self.vbox.addWidget(self.bar) + self.vbox.setAlignment(self.bar, Qt.AlignTop | Qt.AlignLeft) + self.vbox.addWidget(self.bar.view) + self.vbox.setAlignment(self.view, Qt.AlignTop | Qt.AlignLeft) + # self.vbox.addWidget(sel.bar.view) async def handle_keyboard_input( - search: FontSizedQLineEdit, + search: SearchWidget, recv_chan: trio.abc.ReceiveChannel, + keyboard_pause_period: float = 0.0616, ) -> None: global _search_active, _search_enabled # startup - view = search.view - view.set_font_size(search.dpi_font.px_size) + bar = search.bar + view = bar.view + view.set_font_size(bar.dpi_font.px_size) model = view.model() nidx = cidx = view.currentIndex() sel = view.selectionModel() @@ -404,10 +427,13 @@ async def handle_keyboard_input( async with trio.open_nursery() as n: # TODO: async debouncing? n.start_soon( - fill_results, - search, - symsearch, - recv, + partial( + fill_results, + search, + symsearch, + recv, + pause_time=keyboard_pause_period, + ) ) async for key, mods, txt in recv_chan: @@ -438,7 +464,7 @@ async def handle_keyboard_input( _search_enabled = False # release kb control of search bar - search.unfocus() + search.bar.unfocus() continue # selection tips: @@ -450,7 +476,7 @@ async def handle_keyboard_input( if ctrl: # cancel and close if key == Qt.Key_C: - search.unfocus() + search.bar.unfocus() # kill the search and focus back on main chart if search.chart_app: @@ -484,7 +510,7 @@ async def handle_keyboard_input( else: # relay to completer task _search_enabled = True - send.send_nowait(search.text()) + send.send_nowait(search.bar.text()) _search_active.set() @@ -504,7 +530,7 @@ if __name__ == '__main__': # results.setFocusPolicy(Qt.NoFocus) view = CompleterView(['src', 'i', 'symbol']) - search = FontSizedQLineEdit(None, view=view) + search = SearchBar(None, view=view) search.view.set_results(syms) # make a root widget to tie shit together From 51c61587d29bc249f8f9b5c31fc353e8450269df Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 14 May 2021 07:52:27 -0400 Subject: [PATCH 28/81] Make list pop out next to primary y-axis --- piker/ui/_chart.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 7b574422..7f7f7221 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -24,6 +24,7 @@ from types import ModuleType from functools import partial from PyQt5 import QtCore, QtGui +from PyQt5.QtCore import Qt import numpy as np import pyqtgraph as pg import tractor @@ -55,7 +56,7 @@ from ._style import ( _bars_to_left_in_follow_mode, ) from . import _search -from ._search import FontSizedQLineEdit +from ._search import SearchBar, SearchWidget from ._event import open_key_stream from ..data._source import Symbol from ..data._sharedmem import ShmArray @@ -80,20 +81,24 @@ class ChartSpace(QtGui.QWidget): def __init__(self, parent=None): super().__init__(parent) - self.vbox = QtGui.QVBoxLayout(self) + self.hbox = QtGui.QHBoxLayout(self) + self.hbox.setContentsMargins(0, 0, 0, 0) + self.hbox.setSpacing(2) + + self.vbox = QtGui.QVBoxLayout() self.vbox.setContentsMargins(0, 0, 0, 0) self.vbox.setSpacing(2) + self.hbox.addLayout(self.vbox) + self.toolbar_layout = QtGui.QHBoxLayout() self.toolbar_layout.setContentsMargins(0, 0, 0, 0) - self.hbox = QtGui.QHBoxLayout() - self.hbox.setContentsMargins(0, 0, 0, 0) # self.init_timeframes_ui() # self.init_strategy_ui() self.vbox.addLayout(self.toolbar_layout) - self.vbox.addLayout(self.hbox) + # self.vbox.addLayout(self.hbox) self._chart_cache = {} self.linkedcharts: 'LinkedSplitCharts' = None @@ -1595,20 +1600,30 @@ async def _async_main( # setup search widget # search.installEventFilter(self) - search = _search.FontSizedQLineEdit(chart_app) + # search = _search.SearchBar(chart_app) + + search = _search.SearchWidget( + chart_space=chart_app, + ) + # the main chart's view is given focus at startup - search.unfocus() + search.bar.unfocus() # add search singleton to global chart-space widget - chart_app.vbox.addWidget(search) - chart_app.vbox.addWidget(search.view) + chart_app.hbox.addWidget( + search, + + # alights to top and uses minmial space based on + # search bar size hint (i think?) + alignment=Qt.AlignTop + ) chart_app.search = search # this internally starts a ``chart_symbol()`` task above chart_app.load_symbol(brokername, sym, loglevel) async with open_key_stream( - search, + search.bar, ) as key_stream: # start kb handling task for searcher From 25dbe60c778c8a2bd086693befc4bd97f3cd2e95 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 14 May 2021 08:25:32 -0400 Subject: [PATCH 29/81] Flip to ctrl-l to pop out search/list --- piker/ui/_interaction.py | 9 +++++---- piker/ui/order_mode.py | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 4a9771e4..1f543c3e 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -724,7 +724,7 @@ class ChartView(ViewBox): self._key_active = False - def keyPressEvent(self, ev): + def keyPressEvent(self, ev: QtCore.QEvent) -> None: """ This routine should capture key presses in the current view box. @@ -759,10 +759,10 @@ class ChartView(ViewBox): if mods == QtCore.Qt.AltModifier: pass - # ctlr-k - if key == QtCore.Qt.Key_K and ctrl: + # ctlr-l for "lookup" -> open search / lists + if ctrl and key == QtCore.Qt.Key_L: search = self._chart._lc.chart_space.search - search.focus() + search.bar.focus() # esc if key == QtCore.Qt.Key_Escape or (ctrl and key == QtCore.Qt.Key_C): @@ -809,5 +809,6 @@ class ChartView(ViewBox): # elif ev.key() == QtCore.Qt.Key_Backspace: # self.scaleHistory(len(self.axHistory)) else: + # maybe propagate to parent widget ev.ignore() self._key_active = False diff --git a/piker/ui/order_mode.py b/piker/ui/order_mode.py index 07f2b281..22293efa 100644 --- a/piker/ui/order_mode.py +++ b/piker/ui/order_mode.py @@ -402,7 +402,9 @@ async def start_order_mode( # each clearing tick is responded individually elif resp in ('broker_filled',): + action = msg['action'] + # TODO: some kinda progress system order_mode.on_fill( oid, From b39fd5e1fcd09e9861fc6d637f653334d890b974 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 15 May 2021 19:35:52 -0400 Subject: [PATCH 30/81] Use per-provider indented tree layout for results --- piker/ui/_search.py | 131 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 109 insertions(+), 22 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 3d271589..29e91c7c 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -18,6 +18,19 @@ qompleterz: embeddable search and complete using trio, Qt and fuzzywuzzy. """ + +# link set for hackzin on this stuff: +# https://doc.qt.io/qt-5/qheaderview.html#moving-header-sections +# https://doc.qt.io/qt-5/model-view-programming.html +# https://doc.qt.io/qt-5/modelview.html +# https://doc.qt.io/qt-5/qtreeview.html#selectedIndexes +# https://doc.qt.io/qt-5/qmodelindex.html#siblingAtColumn +# https://doc.qt.io/qt-5/qitemselectionmodel.html#currentIndex +# https://www.programcreek.com/python/example/108109/PyQt5.QtWidgets.QTreeView +# https://doc.qt.io/qt-5/qsyntaxhighlighter.html +# https://github.com/qutebrowser/qutebrowser/blob/master/qutebrowser/completion/completiondelegate.py#L243 +# https://forum.qt.io/topic/61343/highlight-matched-substrings-in-qstyleditemdelegate + import sys from functools import partial from typing import ( @@ -37,6 +50,7 @@ from PyQt5.QtCore import ( QItemSelectionModel, ) from PyQt5.QtGui import ( + # QLayout, QStandardItem, QStandardItemModel, ) @@ -44,7 +58,7 @@ from PyQt5.QtWidgets import ( QWidget, QTreeView, # QListWidgetItem, - QAbstractScrollArea, + # QAbstractScrollArea, QStyledItemDelegate, ) @@ -91,6 +105,14 @@ class SimpleDelegate(QStyledItemDelegate): class CompleterView(QTreeView): + # XXX: relevant docs links: + # - simple widget version of this: + # https://doc.qt.io/qt-5/qtreewidget.html#details + # - MVC high level instructional: + # https://doc.qt.io/qt-5/model-view-programming.html + # - MV tut: + # https://doc.qt.io/qt-5/modelview.html + def __init__( self, parent=None, @@ -106,7 +128,8 @@ class CompleterView(QTreeView): self.setItemDelegate(SimpleDelegate()) self.setModel(model) self.setAlternatingRowColors(True) - self.setIndentation(1) + # TODO: size this based on DPI font + self.setIndentation(16) # self.setUniformRowHeights(True) # self.setColumnWidth(0, 3) @@ -117,7 +140,8 @@ class CompleterView(QTreeView): self.setAnimated(False) self.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) - self.setSizeAdjustPolicy(QAbstractScrollArea.AdjustIgnored) + # self.setVerticalBarPolicy(Qt.ScrollBarAlwaysOff) + # self.setSizeAdjustPolicy(QAbstractScrollArea.AdjustIgnored) # column headers model.setHorizontalHeaderLabels(labels) @@ -172,19 +196,26 @@ class CompleterView(QTreeView): # # root index # model.index(0, 0, QModelIndex()), # ) + root = model.invisibleRootItem() + for key, values in results.items(): + src = QStandardItem(key) + root.appendRow(src) + # self.expand(model.index(1, 0, QModelIndex())) + # values just needs to be sequence-like for i, s in enumerate(values): + # blank = QStandardItem('') ix = QStandardItem(str(i)) item = QStandardItem(s) # item.setCheckable(False) - src = QStandardItem(key) - # Add the item to the model - model.appendRow([src, ix, item]) + src.appendRow([ix, item]) + + self.expandAll() def show_matches(self) -> None: # print(f"SHOWING {self}") @@ -199,8 +230,8 @@ class CompleterView(QTreeView): self.resizeColumnToContents(i) # inclusive of search bar and header "rows" in pixel terms - rows = model.rowCount() + 2 - print(f'row count: {rows}') + rows = 100 + # print(f'row count: {rows}') # max_rows = 8 # 6 + search and headers row_px = self.rowHeight(self.currentIndex()) # print(f'font_h: {font_h}\n px_height: {px_height}') @@ -209,6 +240,41 @@ class CompleterView(QTreeView): self.setMinimumSize(self.width(), rows * row_px) self.setMaximumSize(self.width(), rows * row_px) + def select_previous(self) -> QModelIndex: + cidx = self.currentIndex() + nidx = self.indexAbove(cidx) + if nidx.parent() is QModelIndex(): + nidx = self.indexAbove(cidx) + breakpoint() + + return nidx + + def select_next(self) -> QModelIndex: + cidx = self.currentIndex() + nidx = self.indexBelow(cidx) + if nidx.parent() is QModelIndex(): + nidx = self.indexBelow(cidx) + breakpoint() + + return nidx + + def select_from_idx( + self, + idx: QModelIndex, + ) -> None: + sel = self.selectionModel() + model = self.model() + + # select first indented entry + if idx == model.index(0, 0): + idx = self.select_next() + + sel.setCurrentIndex( + idx, + QItemSelectionModel.ClearAndSelect | + QItemSelectionModel.Rows + ) + # def find_matches( # self, # field: str, @@ -241,6 +307,7 @@ class SearchBar(QtWidgets.QLineEdit): self.chart_app = parent_chart # size it as we specify + # https://doc.qt.io/qt-5/qsizepolicy.html#Policy-enum self.setSizePolicy( QtWidgets.QSizePolicy.Fixed, QtWidgets.QSizePolicy.Fixed, @@ -366,12 +433,15 @@ async def fill_results( ) bar.show() + # ensure we select first indented entry + view.select_from_idx(sel.currentIndex()) + class SearchWidget(QtGui.QWidget): def __init__( self, chart_space: 'ChartSpace', # type: ignore # noqa - columns: List[str] = ['src', 'i', 'symbol'], + columns: List[str] = ['src', 'symbol'], parent=None, ): super().__init__(parent or chart_space) @@ -379,13 +449,16 @@ class SearchWidget(QtGui.QWidget): # size it as we specify self.setSizePolicy( QtWidgets.QSizePolicy.Fixed, - QtWidgets.QSizePolicy.Fixed, + QtWidgets.QSizePolicy.Expanding, ) self.chart_app = chart_space self.vbox = QtGui.QVBoxLayout(self) self.vbox.setContentsMargins(0, 0, 0, 0) - self.vbox.setSpacing(2) + self.vbox.setSpacing(4) + + # https://doc.qt.io/qt-5/qlayout.html#SizeConstraint-enum + # self.vbox.setSizeConstraint(QLayout.SetMaximumSize) self.view = CompleterView( parent=self, @@ -400,7 +473,6 @@ class SearchWidget(QtGui.QWidget): self.vbox.setAlignment(self.bar, Qt.AlignTop | Qt.AlignLeft) self.vbox.addWidget(self.bar.view) self.vbox.setAlignment(self.view, Qt.AlignTop | Qt.AlignLeft) - # self.vbox.addWidget(sel.bar.view) async def handle_keyboard_input( @@ -439,7 +511,18 @@ async def handle_keyboard_input( async for key, mods, txt in recv_chan: log.debug(f'key: {key}, mods: {mods}, txt: {txt}') - nidx = cidx = view.currentIndex() + # parent = view.currentIndex() + cidx = sel.currentIndex() + # view.select_from_idx(nidx) + + # if cidx == model.index(0, 0): + # print('uhh') + # cidx = view.select_next() + # sel.setCurrentIndex( + # cidx, + # QItemSelectionModel.ClearAndSelect | + # QItemSelectionModel.Rows + # ) ctrl = False if mods == Qt.ControlModifier: @@ -447,9 +530,12 @@ async def handle_keyboard_input( if key in (Qt.Key_Enter, Qt.Key_Return): - node = model.item(nidx.row(), 2) + # TODO: get rid of this hard coded column -> 1 + # https://doc.qt.io/qt-5/qstandarditemmodel.html#itemFromIndex + node = model.itemFromIndex(cidx.siblingAtColumn(1)) if node: value = node.text() + # print(f' value: {value}') else: continue @@ -489,24 +575,25 @@ async def handle_keyboard_input( _search_enabled = False if key == Qt.Key_K: - nidx = view.indexAbove(cidx) + nidx = view.select_previous() elif key == Qt.Key_J: - nidx = view.indexBelow(cidx) + nidx = view.select_next() # select row without selecting.. :eye_rollzz: # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex if nidx.isValid(): - sel.setCurrentIndex( - nidx, - QItemSelectionModel.ClearAndSelect | - QItemSelectionModel.Rows - ) + view.select_from_idx(nidx) + # sel.setCurrentIndex( + # nidx, + # QItemSelectionModel.ClearAndSelect | + # QItemSelectionModel.Rows + # ) # TODO: make this not hard coded to 2 # and use the ``CompleterView`` schema/settings # to figure out the desired field(s) - value = model.item(nidx.row(), 2).text() + # value = model.item(nidx.row(), 0).text() else: # relay to completer task _search_enabled = True From 82ece83d3347d238eaffe92982a06edc724e12d7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 16 May 2021 15:40:31 -0400 Subject: [PATCH 31/81] Drop commented cruft --- piker/ui/_search.py | 50 ++++++++------------------------------------- 1 file changed, 9 insertions(+), 41 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 29e91c7c..d993bc82 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -39,10 +39,10 @@ from typing import ( ) # from pprint import pformat +from fuzzywuzzy import process as fuzzy +import trio from PyQt5 import QtCore, QtGui from PyQt5 import QtWidgets -import trio - from PyQt5.QtCore import ( Qt, # QSize, @@ -129,7 +129,7 @@ class CompleterView(QTreeView): self.setModel(model) self.setAlternatingRowColors(True) # TODO: size this based on DPI font - self.setIndentation(16) + self.setIndentation(20) # self.setUniformRowHeights(True) # self.setColumnWidth(0, 3) @@ -357,6 +357,7 @@ async def fill_results( search: SearchBar, symsearch: Callable[..., Awaitable], recv_chan: trio.abc.ReceiveChannel, + # cached_symbols: Dict[str, pause_time: float = 0.0616, ) -> None: @@ -477,6 +478,7 @@ class SearchWidget(QtGui.QWidget): async def handle_keyboard_input( + # chart: 'ChartSpace', # type: igore # noqa search: SearchWidget, recv_chan: trio.abc.ReceiveChannel, keyboard_pause_period: float = 0.0616, @@ -513,16 +515,6 @@ async def handle_keyboard_input( log.debug(f'key: {key}, mods: {mods}, txt: {txt}') # parent = view.currentIndex() cidx = sel.currentIndex() - # view.select_from_idx(nidx) - - # if cidx == model.index(0, 0): - # print('uhh') - # cidx = view.select_next() - # sel.setCurrentIndex( - # cidx, - # QItemSelectionModel.ClearAndSelect | - # QItemSelectionModel.Rows - # ) ctrl = False if mods == Qt.ControlModifier: @@ -531,6 +523,8 @@ async def handle_keyboard_input( if key in (Qt.Key_Enter, Qt.Key_Return): # TODO: get rid of this hard coded column -> 1 + # and use the ``CompleterView`` schema/settings + # to figure out the desired field(s) # https://doc.qt.io/qt-5/qstandarditemmodel.html#itemFromIndex node = model.itemFromIndex(cidx.siblingAtColumn(1)) if node: @@ -584,16 +578,6 @@ async def handle_keyboard_input( # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex if nidx.isValid(): view.select_from_idx(nidx) - # sel.setCurrentIndex( - # nidx, - # QItemSelectionModel.ClearAndSelect | - # QItemSelectionModel.Rows - # ) - - # TODO: make this not hard coded to 2 - # and use the ``CompleterView`` schema/settings - # to figure out the desired field(s) - # value = model.item(nidx.row(), 0).text() else: # relay to completer task _search_enabled = True @@ -614,23 +598,7 @@ if __name__ == '__main__': 'XDGUSD', 'ADAUSD', ] - - # results.setFocusPolicy(Qt.NoFocus) - view = CompleterView(['src', 'i', 'symbol']) - search = SearchBar(None, view=view) - search.view.set_results(syms) - - # make a root widget to tie shit together - class W(QtGui.QWidget): - def __init__(self, parent=None): - super().__init__(parent) - self.vbox = QtGui.QVBoxLayout(self) - self.vbox.setContentsMargins(0, 0, 0, 0) - self.vbox.setSpacing(2) - - main = W() - main.vbox.addWidget(search) - main.vbox.addWidget(view) - search.show() + # TODO: need to qtracor.run() here to make it work now... + # search.show() sys.exit(app.exec_()) From 0163a582a52288ef3890f025489a0145a2ab1911 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 16 May 2021 20:52:22 -0400 Subject: [PATCH 32/81] Move search machinery to ui module, add fast cached chart selection --- piker/ui/_search.py | 170 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 144 insertions(+), 26 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index d993bc82..a3922d6b 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -31,11 +31,12 @@ qompleterz: embeddable search and complete using trio, Qt and fuzzywuzzy. # https://github.com/qutebrowser/qutebrowser/blob/master/qutebrowser/completion/completiondelegate.py#L243 # https://forum.qt.io/topic/61343/highlight-matched-substrings-in-qstyleditemdelegate -import sys +from contextlib import asynccontextmanager from functools import partial from typing import ( List, Optional, Callable, Awaitable, Sequence, Dict, + Any, AsyncIterator, Tuple, ) # from pprint import pformat @@ -69,7 +70,6 @@ from ._style import ( DpiAwareFont, # hcolor, ) -from ..data import feed log = get_logger(__name__) @@ -217,6 +217,17 @@ class CompleterView(QTreeView): self.expandAll() + # XXX: these 2 lines MUST be in sequence in order + # to get the view to show right after typing input. + sel = self.selectionModel() + sel.setCurrentIndex( + model.index(0, 0, QModelIndex()), + QItemSelectionModel.ClearAndSelect | + QItemSelectionModel.Rows + ) + self.select_from_idx(model.index(0, 0, QModelIndex())) + + def show_matches(self) -> None: # print(f"SHOWING {self}") self.show() @@ -259,9 +270,12 @@ class CompleterView(QTreeView): return nidx def select_from_idx( + self, idx: QModelIndex, - ) -> None: + + ) -> Tuple[QModelIndex, QStandardItem]: + sel = self.selectionModel() model = self.model() @@ -275,6 +289,8 @@ class CompleterView(QTreeView): QItemSelectionModel.Rows ) + return idx, model.itemFromIndex(idx) + # def find_matches( # self, # field: str, @@ -357,7 +373,7 @@ async def fill_results( search: SearchBar, symsearch: Callable[..., Awaitable], recv_chan: trio.abc.ReceiveChannel, - # cached_symbols: Dict[str, + # cached_symbols: Dict[str, pause_time: float = 0.0616, ) -> None: @@ -427,15 +443,19 @@ async def fill_results( # XXX: these 2 lines MUST be in sequence in order # to get the view to show right after typing input. - sel.setCurrentIndex( - model.index(0, 0, QModelIndex()), - QItemSelectionModel.ClearAndSelect | - QItemSelectionModel.Rows - ) + # ensure we select first indented entry + # view.select_from_idx(model.index(0, 0, QModelIndex())) + + # sel.setCurrentIndex( + # model.index(0, 0, QModelIndex()), + # QItemSelectionModel.ClearAndSelect | + # QItemSelectionModel.Rows + # ) + bar.show() - # ensure we select first indented entry - view.select_from_idx(sel.currentIndex()) + # # ensure we select first indented entry + # view.select_from_idx(sel.currentIndex()) class SearchWidget(QtGui.QWidget): @@ -476,6 +496,13 @@ class SearchWidget(QtGui.QWidget): self.vbox.setAlignment(self.view, Qt.AlignTop | Qt.AlignLeft) + def focus(self) -> None: + # fill cache list + self.view.set_results({'cache': list(self.chart_app._chart_cache)}) + self.bar.focus() + + + async def handle_keyboard_input( # chart: 'ChartSpace', # type: igore # noqa @@ -495,7 +522,7 @@ async def handle_keyboard_input( nidx = cidx = view.currentIndex() sel = view.selectionModel() - symsearch = feed.get_multi_search() + symsearch = get_multi_search() send, recv = trio.open_memory_channel(16) async with trio.open_nursery() as n: @@ -577,7 +604,21 @@ async def handle_keyboard_input( # select row without selecting.. :eye_rollzz: # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex if nidx.isValid(): - view.select_from_idx(nidx) + i, item = view.select_from_idx(nidx) + + if item: + parent_item = item.parent() + if parent_item and parent_item.text() == 'cache': + node = model.itemFromIndex(i.siblingAtColumn(1)) + if node: + value = node.text() + print(f'cache selection') + search.chart_app.load_symbol( + app.linkedcharts.symbol.brokers[0], + value, + 'info', + ) + else: # relay to completer task _search_enabled = True @@ -585,20 +626,97 @@ async def handle_keyboard_input( _search_active.set() -if __name__ == '__main__': +async def search_simple_dict( + text: str, + source: dict, +) -> Dict[str, Any]: + # matches_per_src = {} + + # for source, data in source.items(): + + # search routine can be specified as a function such + # as in the case of the current app's local symbol cache + matches = fuzzy.extractBests( + text, + source.keys(), + score_cutoff=90, + ) + + return [item[0] for item in matches] + + +# cache of provider names to async search routines +_searcher_cache: Dict[str, Callable[..., Awaitable]] = {} + + +def get_multi_search() -> Callable[..., Awaitable]: + + global _searcher_cache + + async def multisearcher( + pattern: str, + ) -> dict: + + matches = {} + + async def pack_matches( + provider: str, + pattern: str, + search: Callable[..., Awaitable[dict]], + ) -> None: + log.debug(f'Searching {provider} for "{pattern}"') + results = await search(pattern) + if results: + matches[provider] = results + + # TODO: make this an async stream? + async with trio.open_nursery() as n: + + for brokername, search in _searcher_cache.items(): + n.start_soon(pack_matches, brokername, pattern, search) + + return matches + + return multisearcher + + +@asynccontextmanager +async def register_symbol_search( + + provider_name: str, + search_routine: Callable, + +) -> AsyncIterator[dict]: + + global _searcher_cache + + # deliver search func to consumer + try: + _searcher_cache[provider_name] = search_routine + yield search_routine + finally: + _searcher_cache.pop(provider_name) + + +# if __name__ == '__main__': + + # TODO: simple standalone widget testing script (moreso + # for if/when we decide to expose this module as a standalone + # repo/project). + + # import sys # local testing of **just** the search UI - app = QtWidgets.QApplication(sys.argv) + # app = QtWidgets.QApplication(sys.argv) - syms = [ - 'XMRUSD', - 'XBTUSD', - 'ETHUSD', - 'XMRXBT', - 'XDGUSD', - 'ADAUSD', - ] - # TODO: need to qtracor.run() here to make it work now... - # search.show() + # syms = [ + # 'XMRUSD', + # 'XBTUSD', + # 'ETHUSD', + # 'XMRXBT', + # 'XDGUSD', + # 'ADAUSD', + # ] + # # search.show() - sys.exit(app.exec_()) + # sys.exit(app.exec_()) From c9c686c98d4640a660fe490453844b41e536c3c3 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 16 May 2021 20:53:21 -0400 Subject: [PATCH 33/81] Register context-stream with multi-search for each feed --- piker/data/feed.py | 100 ++++++++++----------------------------------- 1 file changed, 22 insertions(+), 78 deletions(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index bde8fe72..5300bb85 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -27,7 +27,7 @@ from types import ModuleType from typing import ( Dict, Any, Sequence, AsyncIterator, Optional, - List, Awaitable, Callable + List, Awaitable, Callable, ) import trio @@ -47,6 +47,7 @@ from ._sharedmem import ( ) from .ingest import get_ingestormod from ._source import base_iohlc_dtype, Symbol +from ..ui import _search from ._sampling import ( _shms, _incrementers, @@ -383,78 +384,6 @@ def sym_to_shm_key( return f'{broker}.{symbol}' -# cache of brokernames to feeds -_cache: Dict[str, Callable] = {} -_cache_lock: trio.Lock = trio.Lock() - - -def get_multi_search() -> Callable[..., Awaitable]: - - global _cache - - async def multisearcher( - pattern: str, - ) -> dict: - - matches = {} - - async def pack_matches( - brokername: str, - pattern: str, - search: Callable[..., Awaitable[dict]], - ) -> None: - log.debug(f'Searching {brokername} for "{pattern}"') - matches[brokername] = await search(pattern) - - # TODO: make this an async stream? - async with trio.open_nursery() as n: - - for brokername, search in _cache.items(): - n.start_soon(pack_matches, brokername, pattern, search) - - return matches - - return multisearcher - - -@asynccontextmanager -async def open_symbol_search( - brokermod: ModuleType, - brokerd_portal: tractor._portal.Portal, -) -> AsyncIterator[dict]: - - global _cache - - open_search = getattr(brokermod, 'open_symbol_search', None) - if open_search is None: - - # just return a pure pass through searcher - async def passthru(text: str) -> Dict[str, Any]: - return text - - yield passthru - return - - async with brokerd_portal.open_context( - open_search, - ) as (ctx, cache): - - # shield here since we expect the search rpc to be - # cancellable by the user as they see fit. - async with ctx.open_stream() as stream: - - async def search(text: str) -> Dict[str, Any]: - await stream.send(text) - return await stream.receive() - - # deliver search func to consumer - try: - _cache[brokermod.name] = search - yield search - finally: - _cache.pop(brokermod.name) - - @asynccontextmanager async def open_feed( brokername: str, @@ -464,15 +393,15 @@ async def open_feed( """Open a "data feed" which provides streamed real-time quotes. """ - global _cache, _cache_lock sym = symbols[0].lower() # TODO: feed cache locking, right now this is causing # issues when reconncting to a long running emsd? + # global _searcher_cache # async with _cache_lock: - # feed = _cache.get((brokername, sym)) + # feed = _searcher_cache.get((brokername, sym)) # # if feed is not None and sym in feed.symbols: # if feed is not None: @@ -542,8 +471,23 @@ async def open_feed( feed._max_sample_rate = max(ohlc_sample_rates) - if brokername in _cache: + if brokername in _search._searcher_cache: yield feed else: - async with open_symbol_search(mod, feed._brokerd_portal): - yield feed + async with feed._brokerd_portal.open_context( + mod.open_symbol_search + ) as (ctx, cache): + + # shield here since we expect the search rpc to be + # cancellable by the user as they see fit. + async with ctx.open_stream() as stream: + + async def search(text: str) -> Dict[str, Any]: + await stream.send(text) + return await stream.receive() + + async with _search.register_symbol_search( + provider_name=brokername, + search_routine=search, + ): + yield feed From cb102f692c09c34cb4127a4d0a2f37856f43dd8e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 16 May 2021 20:53:51 -0400 Subject: [PATCH 34/81] Top level widget `.focus()` --- piker/ui/_interaction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 1f543c3e..05adfbcd 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -188,7 +188,7 @@ class SelectRect(QtGui.QGraphicsRectItem): self._abs_top_right = label_anchor self._label_proxy.setPos(self.vb.mapFromView(label_anchor)) - self._label.show() + # self._label.show() def clear(self): """Clear the selection box from view. @@ -762,7 +762,7 @@ class ChartView(ViewBox): # ctlr-l for "lookup" -> open search / lists if ctrl and key == QtCore.Qt.Key_L: search = self._chart._lc.chart_space.search - search.bar.focus() + search.focus() # esc if key == QtCore.Qt.Key_Escape or (ctrl and key == QtCore.Qt.Key_C): From 63363d750cd5cfa5b894227ba83f8abb17bae262 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 16 May 2021 20:54:56 -0400 Subject: [PATCH 35/81] Port chart to multi-search api --- piker/ui/_chart.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 7f7f7221..330d4d14 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -56,13 +56,12 @@ from ._style import ( _bars_to_left_in_follow_mode, ) from . import _search -from ._search import SearchBar, SearchWidget from ._event import open_key_stream from ..data._source import Symbol from ..data._sharedmem import ShmArray +from ..data import maybe_open_shm_array from .. import brokers from .. import data -from ..data import maybe_open_shm_array from ..log import get_logger from ._exec import run_qtractor, current_screen from ._interaction import ChartView @@ -94,7 +93,6 @@ class ChartSpace(QtGui.QWidget): self.toolbar_layout = QtGui.QHBoxLayout() self.toolbar_layout.setContentsMargins(0, 0, 0, 0) - # self.init_timeframes_ui() # self.init_strategy_ui() self.vbox.addLayout(self.toolbar_layout) @@ -182,6 +180,7 @@ class ChartSpace(QtGui.QWidget): self.linkedcharts = linkedcharts symbol = linkedcharts.symbol + if symbol is not None: self.window.setWindowTitle( f'{symbol.key}@{symbol.brokers} ' @@ -1600,8 +1599,6 @@ async def _async_main( # setup search widget # search.installEventFilter(self) - # search = _search.SearchBar(chart_app) - search = _search.SearchWidget( chart_space=chart_app, ) @@ -1622,18 +1619,28 @@ async def _async_main( # this internally starts a ``chart_symbol()`` task above chart_app.load_symbol(brokername, sym, loglevel) - async with open_key_stream( - search.bar, - ) as key_stream: + async with _search.register_symbol_search( - # start kb handling task for searcher - root_n.start_soon( - _search.handle_keyboard_input, - search, - key_stream, - ) + provider_name='cache', + search_routine=partial( + _search.search_simple_dict, + source=chart_app._chart_cache, + ), - await trio.sleep_forever() + ): + async with open_key_stream( + search.bar, + ) as key_stream: + + # start kb handling task for searcher + root_n.start_soon( + _search.handle_keyboard_input, + # chart_app, + search, + key_stream, + ) + + await trio.sleep_forever() def _main( From b2ff09f1937427e35b892d767cc5def8e92c10f0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 18 May 2021 08:19:52 -0400 Subject: [PATCH 36/81] Support min and max keyboard pauses Some providers do well with a "longer" debounce period (like ib) since searching them too frequently causes latency and stalls. By supporting both a min and max debounce period on keyboard input we can only send patterns to the slower engines when that period is triggered via `trio.move_on_after()` and continue to relay to faster engines when the measured period permits. Allow search routines to register their "min period" such that they can choose to ignore patterns that arrive before their heuristically known ideal wait. --- piker/ui/_search.py | 145 ++++++++++++++++++++++---------------------- 1 file changed, 73 insertions(+), 72 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index a3922d6b..79c545e0 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -38,6 +38,7 @@ from typing import ( Awaitable, Sequence, Dict, Any, AsyncIterator, Tuple, ) +import time # from pprint import pformat from fuzzywuzzy import process as fuzzy @@ -207,7 +208,6 @@ class CompleterView(QTreeView): # values just needs to be sequence-like for i, s in enumerate(values): - # blank = QStandardItem('') ix = QStandardItem(str(i)) item = QStandardItem(s) # item.setCheckable(False) @@ -220,16 +220,20 @@ class CompleterView(QTreeView): # XXX: these 2 lines MUST be in sequence in order # to get the view to show right after typing input. sel = self.selectionModel() + + # select row without selecting.. :eye_rollzz: + # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex sel.setCurrentIndex( model.index(0, 0, QModelIndex()), QItemSelectionModel.ClearAndSelect | QItemSelectionModel.Rows ) + + # ensure we're **not** selecting the first level parent node and + # instead its child. self.select_from_idx(model.index(0, 0, QModelIndex())) - def show_matches(self) -> None: - # print(f"SHOWING {self}") self.show() self.resize() @@ -242,7 +246,6 @@ class CompleterView(QTreeView): # inclusive of search bar and header "rows" in pixel terms rows = 100 - # print(f'row count: {rows}') # max_rows = 8 # 6 + search and headers row_px = self.rowHeight(self.currentIndex()) # print(f'font_h: {font_h}\n px_height: {px_height}') @@ -374,7 +377,11 @@ async def fill_results( symsearch: Callable[..., Awaitable], recv_chan: trio.abc.ReceiveChannel, # cached_symbols: Dict[str, - pause_time: float = 0.0616, + + # kb debouncing pauses + min_pause_time: float = 0.0616, + # long_pause_time: float = 0.4, + max_pause_time: float = 6/16, ) -> None: """Task to search through providers and fill in possible @@ -385,77 +392,62 @@ async def fill_results( bar = search.bar view = bar.view - sel = bar.view.selectionModel() - model = bar.view.model() - last_search_text = '' last_text = bar.text() repeats = 0 while True: - - last_text = bar.text() await _search_active.wait() + period = None - with trio.move_on_after(pause_time) as cs: - # cs.shield = True - pattern = await recv_chan.receive() - print(pattern) + while True: - # during fast multiple key inputs, wait until a pause - # (in typing) to initiate search - if not cs.cancelled_caught: - log.debug(f'Ignoring fast input for {pattern}') - continue + last_text = bar.text() + wait_start = time.time() - text = bar.text() - print(f'search: {text}') + with trio.move_on_after(max_pause_time): + pattern = await recv_chan.receive() - if not text: - print('idling') - _search_active = trio.Event() - continue + period = time.time() - wait_start + print(f'{pattern} after {period}') - if text == last_text: - repeats += 1 + # during fast multiple key inputs, wait until a pause + # (in typing) to initiate search + if period < min_pause_time: + log.debug(f'Ignoring fast input for {pattern}') + continue - if repeats > 1: - _search_active = trio.Event() - repeats = 0 + text = bar.text() + print(f'search: {text}') - if not _search_enabled: - print('search not ENABLED?') - continue + if not text: + print('idling') + _search_active = trio.Event() + break - if last_search_text and last_search_text == text: - continue + if repeats > 2 and period >= max_pause_time: + _search_active = trio.Event() + repeats = 0 + break - log.debug(f'Search req for {text}') + if text == last_text: + repeats += 1 - last_search_text = text - results = await symsearch(text) - log.debug(f'Received search result {results}') + if not _search_enabled: + print('search currently disabled') + break - if results and _search_enabled: + log.debug(f'Search req for {text}') - # TODO: indented branch results for each provider - view.set_results(results) + results = await symsearch(text, period=period) - # XXX: these 2 lines MUST be in sequence in order - # to get the view to show right after typing input. - # ensure we select first indented entry - # view.select_from_idx(model.index(0, 0, QModelIndex())) + log.debug(f'Received search result {results}') - # sel.setCurrentIndex( - # model.index(0, 0, QModelIndex()), - # QItemSelectionModel.ClearAndSelect | - # QItemSelectionModel.Rows - # ) + if results and _search_enabled: - bar.show() - - # # ensure we select first indented entry - # view.select_from_idx(sel.currentIndex()) + # show the results in the completer view + view.set_results(results) + bar.show() class SearchWidget(QtGui.QWidget): @@ -491,21 +483,18 @@ class SearchWidget(QtGui.QWidget): view=self.view, ) self.vbox.addWidget(self.bar) - self.vbox.setAlignment(self.bar, Qt.AlignTop | Qt.AlignLeft) + self.vbox.setAlignment(self.bar, Qt.AlignTop | Qt.AlignRight) self.vbox.addWidget(self.bar.view) self.vbox.setAlignment(self.view, Qt.AlignTop | Qt.AlignLeft) - def focus(self) -> None: # fill cache list self.view.set_results({'cache': list(self.chart_app._chart_cache)}) self.bar.focus() - async def handle_keyboard_input( - # chart: 'ChartSpace', # type: igore # noqa search: SearchWidget, recv_chan: trio.abc.ReceiveChannel, keyboard_pause_period: float = 0.0616, @@ -533,7 +522,6 @@ async def handle_keyboard_input( search, symsearch, recv, - pause_time=keyboard_pause_period, ) ) @@ -601,21 +589,24 @@ async def handle_keyboard_input( elif key == Qt.Key_J: nidx = view.select_next() - # select row without selecting.. :eye_rollzz: - # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex if nidx.isValid(): i, item = view.select_from_idx(nidx) if item: parent_item = item.parent() if parent_item and parent_item.text() == 'cache': - node = model.itemFromIndex(i.siblingAtColumn(1)) + node = model.itemFromIndex( + i.siblingAtColumn(1) + ) if node: + + # TODO: parse out provider from + # cached value. value = node.text() - print(f'cache selection') + search.chart_app.load_symbol( app.linkedcharts.symbol.brokers[0], - value, + node.text(), 'info', ) @@ -631,10 +622,6 @@ async def search_simple_dict( source: dict, ) -> Dict[str, Any]: - # matches_per_src = {} - - # for source, data in source.items(): - # search routine can be specified as a function such # as in the case of the current app's local symbol cache matches = fuzzy.extractBests( @@ -656,6 +643,8 @@ def get_multi_search() -> Callable[..., Awaitable]: async def multisearcher( pattern: str, + period: str, + ) -> dict: matches = {} @@ -664,7 +653,9 @@ def get_multi_search() -> Callable[..., Awaitable]: provider: str, pattern: str, search: Callable[..., Awaitable[dict]], + ) -> None: + log.debug(f'Searching {provider} for "{pattern}"') results = await search(pattern) if results: @@ -673,8 +664,14 @@ def get_multi_search() -> Callable[..., Awaitable]: # TODO: make this an async stream? async with trio.open_nursery() as n: - for brokername, search in _searcher_cache.items(): - n.start_soon(pack_matches, brokername, pattern, search) + for provider, (search, min_pause) in _searcher_cache.items(): + + # only conduct search on this backend if it's registered + # for the corresponding pause period. + if period >= min_pause: + # print( + # f'searching {provider} after {period} > {min_pause}') + n.start_soon(pack_matches, provider, pattern, search) return matches @@ -686,15 +683,19 @@ async def register_symbol_search( provider_name: str, search_routine: Callable, + pause_period: Optional[float] = None, ) -> AsyncIterator[dict]: global _searcher_cache + pause_period = pause_period or 0.061 + # deliver search func to consumer try: - _searcher_cache[provider_name] = search_routine + _searcher_cache[provider_name] = (search_routine, pause_period) yield search_routine + finally: _searcher_cache.pop(provider_name) From bbd5883e5238ff8a0f9540aff420bfc4f52929b2 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 18 May 2021 08:35:39 -0400 Subject: [PATCH 37/81] Add search pause configs to backends --- piker/brokers/ib.py | 35 ++++++++++++++++++++++++----------- piker/brokers/kraken.py | 11 ++++++++--- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index eea26b62..c56a70a8 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -89,7 +89,15 @@ _time_frames = { 'Y': 'OneYear', } -_show_wap_in_history = False +_show_wap_in_history: bool = False + +# optional search config the backend can register for +# it's symbol search handling (in this case we avoid +# accepting patterns before the kb has settled more then +# a quarter second). +_search_conf = { + 'pause_period': 6/16, +} # overrides to sidestep pretty questionable design decisions in @@ -156,7 +164,7 @@ _adhoc_futes_set = { 'mes.globex', } - # https://misc.interactivebrokers.com/cstools/contract_info/v3.10/index.php?action=Conid%20Info&wlId=IB&conid=69067924 +# https://misc.interactivebrokers.com/cstools/contract_info/v3.10/index.php?action=Conid%20Info&wlId=IB&conid=69067924 _enters = 0 @@ -875,7 +883,16 @@ async def backfill_bars( i = 0 while i < count: - bars, bars_array, next_dt = await get_bars(sym, end_dt=next_dt) + out = await get_bars(sym, end_dt=next_dt) + + if out is None: + # could be trying to retreive bars over weekend + # TODO: add logic here to handle tradable hours and only grab + # valid bars in the range + log.error(f"Can't grab bars starting at {next_dt}!?!?") + continue + + bars, bars_array, next_dt = out shm.push(bars_array, prepend=True) i += 1 @@ -1255,6 +1272,10 @@ async def open_symbol_search( except trio.WouldBlock: pass + if not pattern: + log.warning(f'empty pattern received, skipping..') + continue + log.debug(f'searching for {pattern}') # await tractor.breakpoint() last = time.time() @@ -1264,14 +1285,6 @@ async def open_symbol_search( upto=5, ) log.debug(f'got results {results.keys()}') - # results = await client.search_stocks( - # pattern=pattern, upto=5) - - # if cs.cancelled_caught: - # print(f'timed out search for {pattern} !?') - # # await tractor.breakpoint() - # await stream.send({}) - # continue log.debug("fuzzy matching") matches = fuzzy.extractBests( diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index 09a92c90..62fc6115 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -57,6 +57,11 @@ log = get_logger(__name__) _url = 'https://api.kraken.com/0' +_search_conf = { + 'pause_period': 0.0616 +} + + # Broker specific ohlc schema which includes a vwap field _ohlc_dtype = [ ('index', int), @@ -231,6 +236,7 @@ class Client: if since is None: since = arrow.utcnow().floor('minute').shift( minutes=-count).timestamp() + # UTC 2017-07-02 12:53:20 is oldest seconds value since = str(max(1499000000, since)) json = await self._public( @@ -488,12 +494,12 @@ async def open_autorecon_ws(url): async def backfill_bars( + sym: str, shm: ShmArray, # type: ignore # noqa - count: int = 10, # NOTE: any more and we'll overrun the underlying buffer - task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED, + ) -> None: """Fill historical bars into shared mem / storage afap. """ @@ -639,7 +645,6 @@ async def stream_quotes( await send_chan.send({topic: quote}) - @tractor.context async def open_symbol_search( ctx: tractor.Context, From 59377da0ad9d5af5862fdfd4d578651ee47e7765 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 18 May 2021 08:36:19 -0400 Subject: [PATCH 38/81] Load pause configs from backends on feed opens --- piker/data/feed.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/piker/data/feed.py b/piker/data/feed.py index 5300bb85..ce26d6a2 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -473,6 +473,7 @@ async def open_feed( if brokername in _search._searcher_cache: yield feed + else: async with feed._brokerd_portal.open_context( mod.open_symbol_search @@ -489,5 +490,7 @@ async def open_feed( async with _search.register_symbol_search( provider_name=brokername, search_routine=search, + pause_period=mod._search_conf.get('pause_period'), + ): yield feed From fd8dc4f1a3389b35e8251e3f5ca1d6045a2da4fc Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 18 May 2021 08:38:13 -0400 Subject: [PATCH 39/81] Make -b a multi-option for backends --- piker/brokers/cli.py | 14 +++++++------- piker/cli/__init__.py | 20 +++++++++++++++----- piker/ui/_chart.py | 8 ++++---- piker/ui/_exec.py | 1 + piker/ui/cli.py | 6 +++--- 5 files changed, 30 insertions(+), 19 deletions(-) diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py index aa3abd24..b0083cfa 100644 --- a/piker/brokers/cli.py +++ b/piker/brokers/cli.py @@ -50,7 +50,7 @@ def api(config, meth, kwargs, keys): """Make a broker-client API method call """ # global opts - broker = config['broker'] + broker = config['brokers'][0] _kwargs = {} for kwarg in kwargs: @@ -87,7 +87,7 @@ def quote(config, tickers, df_output): """Print symbol quotes to the console """ # global opts - brokermod = config['brokermod'] + brokermod = config['brokermods'][0] quotes = trio.run(partial(core.stocks_quote, brokermod, tickers)) if not quotes: @@ -123,7 +123,7 @@ def bars(config, symbol, count, df_output): """Retreive 1m bars for symbol and print on the console """ # global opts - brokermod = config['brokermod'] + brokermod = config['brokermods'][0] # broker backend should return at the least a # list of candle dictionaries @@ -159,7 +159,7 @@ def record(config, rate, name, dhost, filename): """Record client side quotes to a file on disk """ # global opts - brokermod = config['brokermod'] + brokermod = config['brokermods'][0] loglevel = config['loglevel'] log = config['log'] @@ -222,7 +222,7 @@ def optsquote(config, symbol, df_output, date): """Retreive symbol option quotes on the console """ # global opts - brokermod = config['brokermod'] + brokermod = config['brokermods'][0] quotes = trio.run( partial( @@ -250,7 +250,7 @@ def symbol_info(config, tickers): """Print symbol quotes to the console """ # global opts - brokermod = config['brokermod'] + brokermod = config['brokermods'][0] quotes = trio.run(partial(core.symbol_info, brokermod, tickers)) if not quotes: @@ -273,7 +273,7 @@ def search(config, pattern): """Search for symbols from broker backend(s). """ # global opts - brokermod = config['brokermod'] + brokermod = config['brokermods'][0] quotes = tractor.run( partial(core.symbol_search, brokermod, pattern), diff --git a/piker/cli/__init__.py b/piker/cli/__init__.py index 2ca5a1a6..0cc38874 100644 --- a/piker/cli/__init__.py +++ b/piker/cli/__init__.py @@ -57,21 +57,31 @@ def pikerd(loglevel, host, tl, pdb): @click.group(context_settings=_context_defaults) -@click.option('--broker', '-b', default=DEFAULT_BROKER, - help='Broker backend to use') +@click.option( + '--brokers', '-b', + default=[DEFAULT_BROKER], + multiple=True, + help='Broker backend to use' +) @click.option('--loglevel', '-l', default='warning', help='Logging level') @click.option('--tl', is_flag=True, help='Enable tractor logging') @click.option('--configdir', '-c', help='Configuration directory') @click.pass_context -def cli(ctx, broker, loglevel, tl, configdir): +def cli(ctx, brokers, loglevel, tl, configdir): if configdir is not None: assert os.path.isdir(configdir), f"`{configdir}` is not a valid path" config._override_config_dir(configdir) ctx.ensure_object(dict) + + if len(brokers) == 1: + brokermods = [get_brokermod(brokers[0])] + else: + brokermods = [get_brokermod(broker) for broker in brokers] + ctx.obj.update({ - 'broker': broker, - 'brokermod': get_brokermod(broker), + 'brokers': brokers, + 'brokermods': brokermods, 'loglevel': loglevel, 'tractorloglevel': None, 'log': get_console_log(loglevel), diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 330d4d14..a0e2628e 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1563,7 +1563,7 @@ async def _async_main( widgets: Dict[str, Any], sym: str, - brokername: str, + brokernames: str, loglevel: str, ) -> None: @@ -1617,7 +1617,7 @@ async def _async_main( chart_app.search = search # this internally starts a ``chart_symbol()`` task above - chart_app.load_symbol(brokername, sym, loglevel) + chart_app.load_symbol(brokernames[0], sym, loglevel) async with _search.register_symbol_search( @@ -1645,7 +1645,7 @@ async def _async_main( def _main( sym: str, - brokername: str, + brokernames: [str], piker_loglevel: str, tractor_kwargs, ) -> None: @@ -1655,7 +1655,7 @@ def _main( # Qt entry point run_qtractor( func=_async_main, - args=(sym, brokername, piker_loglevel), + args=(sym, brokernames, piker_loglevel), main_widget=ChartSpace, tractor_kwargs=tractor_kwargs, ) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index 53de8554..96755fab 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -38,6 +38,7 @@ from PyQt5.QtCore import ( QCoreApplication, ) import qdarkstyle +# import qdarkgraystyle import trio import tractor from outcome import Error diff --git a/piker/ui/cli.py b/piker/ui/cli.py index b40a4c31..b674acfe 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -49,7 +49,7 @@ def monitor(config, rate, name, dhost, test, tl): """Start a real-time watchlist UI """ # global opts - brokermod = config['brokermod'] + brokermod = config['brokermods'][0] loglevel = config['loglevel'] log = config['log'] @@ -142,13 +142,13 @@ def chart(config, symbol, profile, pdb): _profile._pg_profile = profile # global opts - brokername = config['broker'] + brokernames = config['brokers'] tractorloglevel = config['tractorloglevel'] pikerloglevel = config['loglevel'] _main( sym=symbol, - brokername=brokername, + brokernames=brokernames, piker_loglevel=pikerloglevel, tractor_kwargs={ 'debug_mode': pdb, From 1bd0ee87467859a702e728a680fd893e3d2a67b7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 18 May 2021 11:22:29 -0400 Subject: [PATCH 40/81] Support loading multi-brokerds search at startup --- piker/data/feed.py | 62 ++++++++++++++++++++++++-------- piker/ui/_chart.py | 89 +++++++++++++++++++++++++++++++++++----------- 2 files changed, 115 insertions(+), 36 deletions(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index ce26d6a2..5a78c740 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -384,6 +384,32 @@ def sym_to_shm_key( return f'{broker}.{symbol}' +@asynccontextmanager +async def install_brokerd_search( + portal: tractor._portal.Portal, + brokermod: ModuleType, +) -> None: + async with portal.open_context( + brokermod.open_symbol_search + ) as (ctx, cache): + + # shield here since we expect the search rpc to be + # cancellable by the user as they see fit. + async with ctx.open_stream() as stream: + + async def search(text: str) -> Dict[str, Any]: + await stream.send(text) + return await stream.receive() + + async with _search.register_symbol_search( + provider_name=brokermod.name, + search_routine=search, + pause_period=brokermod._search_conf.get('pause_period'), + + ): + yield + + @asynccontextmanager async def open_feed( brokername: str, @@ -475,22 +501,28 @@ async def open_feed( yield feed else: - async with feed._brokerd_portal.open_context( - mod.open_symbol_search - ) as (ctx, cache): + async with install_brokerd_search( + feed._brokerd_portal, + mod, + ): + yield feed - # shield here since we expect the search rpc to be - # cancellable by the user as they see fit. - async with ctx.open_stream() as stream: + # async with feed._brokerd_portal.open_context( + # mod.open_symbol_search + # ) as (ctx, cache): - async def search(text: str) -> Dict[str, Any]: - await stream.send(text) - return await stream.receive() + # # shield here since we expect the search rpc to be + # # cancellable by the user as they see fit. + # async with ctx.open_stream() as stream: - async with _search.register_symbol_search( - provider_name=brokername, - search_routine=search, - pause_period=mod._search_conf.get('pause_period'), + # async def search(text: str) -> Dict[str, Any]: + # await stream.send(text) + # return await stream.receive() - ): - yield feed + # async with _search.register_symbol_search( + # provider_name=brokername, + # search_routine=search, + # pause_period=mod._search_conf.get('pause_period'), + + # ): + # yield feed diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index a0e2628e..427add22 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -22,6 +22,7 @@ import time from typing import Tuple, Dict, Any, Optional, Callable from types import ModuleType from functools import partial +from contextlib import AsyncExitStack from PyQt5 import QtCore, QtGui from PyQt5.QtCore import Qt @@ -30,6 +31,10 @@ import pyqtgraph as pg import tractor import trio +from .._daemon import ( + maybe_spawn_brokerd, +) +from ..brokers import get_brokermod from ._axes import ( DynamicDateAxis, PriceAxis, @@ -67,6 +72,7 @@ from ._exec import run_qtractor, current_screen from ._interaction import ChartView from .order_mode import start_order_mode from .. import fsp +from ..data import feed log = get_logger(__name__) @@ -104,6 +110,19 @@ class ChartSpace(QtGui.QWidget): self._root_n: Optional[trio.Nursery] = None + def set_chart_symbol( + self, + symbol_key: str, # of form . + linked_charts: 'LinkedSplitCharts', # type: ignore + ) -> None: + self._chart_cache[symbol_key] = linked_charts + + def get_chart_symbol( + self, + symbol_key: str, + ) -> 'LinkedSplitCharts': # type: ignore + return self._chart_cache.get(symbol_key) + def init_timeframes_ui(self): self.tf_layout = QtGui.QHBoxLayout() self.tf_layout.setSpacing(0) @@ -128,7 +147,7 @@ class ChartSpace(QtGui.QWidget): def load_symbol( self, - brokername: str, + providername: str, symbol_key: str, loglevel: str, ohlc: bool = True, @@ -142,7 +161,10 @@ class ChartSpace(QtGui.QWidget): # our symbol key style is always lower case symbol_key = symbol_key.lower() - linkedcharts = self._chart_cache.get(symbol_key) + # fully qualified symbol name (SNS i guess is what we're making?) + fqsn = '.'.join([symbol_key, providername]) + + linkedcharts = self.get_chart_symbol(fqsn) if not self.vbox.isEmpty(): # XXX: this is CRITICAL especially with pixel buffer caching @@ -162,13 +184,13 @@ class ChartSpace(QtGui.QWidget): self._root_n.start_soon( chart_symbol, self, - brokername, + providername, symbol_key, loglevel, ) self.vbox.addWidget(linkedcharts) - self._chart_cache[symbol_key] = linkedcharts + self.set_chart_symbol(fqsn, linkedcharts) # chart is already in memory so just focus it if self.linkedcharts: @@ -1619,28 +1641,53 @@ async def _async_main( # this internally starts a ``chart_symbol()`` task above chart_app.load_symbol(brokernames[0], sym, loglevel) - async with _search.register_symbol_search( + # TODO: seems like our incentive for brokerd caching lelel + backends = {} - provider_name='cache', - search_routine=partial( - _search.search_simple_dict, - source=chart_app._chart_cache, - ), + async with AsyncExitStack() as stack: - ): - async with open_key_stream( - search.bar, - ) as key_stream: + # TODO: spawn these async in nursery. - # start kb handling task for searcher - root_n.start_soon( - _search.handle_keyboard_input, - # chart_app, - search, - key_stream, + # load all requested brokerd's at startup and load their + # search engines. + for broker in brokernames: + portal = await stack.enter_async_context( + maybe_spawn_brokerd( + broker, + loglevel=loglevel + ) ) - await trio.sleep_forever() + backends[broker] = portal + await stack.enter_async_context( + feed.install_brokerd_search( + portal, + get_brokermod(broker), + ) + ) + + async with _search.register_symbol_search( + + provider_name='cache', + search_routine=partial( + _search.search_simple_dict, + source=chart_app._chart_cache, + ), + + ): + async with open_key_stream( + search.bar, + ) as key_stream: + + # start kb handling task for searcher + root_n.start_soon( + _search.handle_keyboard_input, + # chart_app, + search, + key_stream, + ) + + await trio.sleep_forever() def _main( From e77a51f16e3f199e9a445b429151a8aca7d5056b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 18 May 2021 11:22:59 -0400 Subject: [PATCH 41/81] Support multi-provider cache symbol switching --- piker/ui/_search.py | 91 +++++++++++++++++++++++++++++++++------------ 1 file changed, 67 insertions(+), 24 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 79c545e0..96143617 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -492,6 +492,34 @@ class SearchWidget(QtGui.QWidget): self.view.set_results({'cache': list(self.chart_app._chart_cache)}) self.bar.focus() + def get_current(self) -> Optional[Tuple[str, str]]: + '''Return the current completer tree selection as + a tuple ``(parent: str, child: str)`` if valid, else ``None``. + + + ''' + model = self.view.model() + sel = self.view.selectionModel() + cidx = sel.currentIndex() + + # TODO: get rid of this hard coded column -> 1 + # and use the ``CompleterView`` schema/settings + # to figure out the desired field(s) + # https://doc.qt.io/qt-5/qstandarditemmodel.html#itemFromIndex + node = model.itemFromIndex(cidx.siblingAtColumn(1)) + if node: + symbol = node.text() + provider = node.parent().text() + + # TODO: move this to somewhere non-search machinery specific? + if provider == 'cache': + symbol, _, provider = symbol.rpartition('.') + + return provider, symbol + + else: + return None + async def handle_keyboard_input( @@ -507,9 +535,10 @@ async def handle_keyboard_input( bar = search.bar view = bar.view view.set_font_size(bar.dpi_font.px_size) - model = view.model() - nidx = cidx = view.currentIndex() - sel = view.selectionModel() + # model = view.model() + # nidx = cidx = view.currentIndex() + nidx = view.currentIndex() + # sel = view.selectionModel() symsearch = get_multi_search() send, recv = trio.open_memory_channel(16) @@ -529,7 +558,7 @@ async def handle_keyboard_input( log.debug(f'key: {key}, mods: {mods}, txt: {txt}') # parent = view.currentIndex() - cidx = sel.currentIndex() + # cidx = sel.currentIndex() ctrl = False if mods == Qt.ControlModifier: @@ -537,23 +566,30 @@ async def handle_keyboard_input( if key in (Qt.Key_Enter, Qt.Key_Return): - # TODO: get rid of this hard coded column -> 1 - # and use the ``CompleterView`` schema/settings - # to figure out the desired field(s) - # https://doc.qt.io/qt-5/qstandarditemmodel.html#itemFromIndex - node = model.itemFromIndex(cidx.siblingAtColumn(1)) - if node: - value = node.text() - # print(f' value: {value}') - else: + value = search.get_current() + if value is None: continue - log.info(f'Requesting symbol: {value}') + provider, symbol = value - app = search.chart_app + # # TODO: get rid of this hard coded column -> 1 + # # and use the ``CompleterView`` schema/settings + # # to figure out the desired field(s) + # # https://doc.qt.io/qt-5/qstandarditemmodel.html#itemFromIndex + # node = model.itemFromIndex(cidx.siblingAtColumn(1)) + # if node: + # symbol = node.text() + # provider = node.parent().text() + # # print(f' value: {value}') + # else: + # continue + + log.info(f'Requesting symbol: {symbol}.{provider}') + + # app = search.chart_app search.chart_app.load_symbol( - app.linkedcharts.symbol.brokers[0], - value, + provider, + symbol, 'info', ) @@ -593,20 +629,27 @@ async def handle_keyboard_input( i, item = view.select_from_idx(nidx) if item: + parent_item = item.parent() if parent_item and parent_item.text() == 'cache': - node = model.itemFromIndex( - i.siblingAtColumn(1) - ) - if node: + + value = search.get_current() + if value is not None: + # continue + + provider, symbol = value + # node = model.itemFromIndex( + # i.siblingAtColumn(1) + # ) + # if node: # TODO: parse out provider from # cached value. - value = node.text() + # value = node.text() search.chart_app.load_symbol( - app.linkedcharts.symbol.brokers[0], - node.text(), + provider, + symbol, 'info', ) From 42fda2a9e62bc162e92a1ef2e9b156980371db88 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 18 May 2021 11:57:42 -0400 Subject: [PATCH 42/81] Drop old code --- piker/data/feed.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index 5a78c740..012df910 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -506,23 +506,3 @@ async def open_feed( mod, ): yield feed - - # async with feed._brokerd_portal.open_context( - # mod.open_symbol_search - # ) as (ctx, cache): - - # # shield here since we expect the search rpc to be - # # cancellable by the user as they see fit. - # async with ctx.open_stream() as stream: - - # async def search(text: str) -> Dict[str, Any]: - # await stream.send(text) - # return await stream.receive() - - # async with _search.register_symbol_search( - # provider_name=brokername, - # search_routine=search, - # pause_period=mod._search_conf.get('pause_period'), - - # ): - # yield feed From 6f3b7999608b4cbeb972f1152bc8d64dd12c3c4b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 18 May 2021 11:58:46 -0400 Subject: [PATCH 43/81] Skip ib exchanges we haven't tested yet --- piker/brokers/ib.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index c56a70a8..d6ed1ba7 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -164,6 +164,14 @@ _adhoc_futes_set = { 'mes.globex', } +# exchanges we don't support at the moment due to not knowing +# how to do symbol-contract lookup correctly likely due +# to not having the data feeds subscribed. +_exch_skip_list = { + 'ASX', # aussie stocks + 'MEXI', # mexican stocks +} + # https://misc.interactivebrokers.com/cstools/contract_info/v3.10/index.php?action=Conid%20Info&wlId=IB&conid=69067924 _enters = 0 @@ -266,10 +274,12 @@ class Client: descriptions = await self.ib.reqMatchingSymbolsAsync(pattern) if descriptions is not None: + futs = [] for d in descriptions: con = d.contract - futs.append(self.ib.reqContractDetailsAsync(con)) + if con.primaryExchange not in _exch_skip_list: + futs.append(self.ib.reqContractDetailsAsync(con)) # batch request all details results = await asyncio.gather(*futs) @@ -1273,7 +1283,7 @@ async def open_symbol_search( pass if not pattern: - log.warning(f'empty pattern received, skipping..') + log.warning('empty pattern received, skipping..') continue log.debug(f'searching for {pattern}') From c9311dd7d0d61faad57eabb4ee3959cd911cf9b6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 18 May 2021 12:33:03 -0400 Subject: [PATCH 44/81] Few more derivs symbols --- piker/brokers/ib.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index d6ed1ba7..289d0583 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -158,10 +158,19 @@ _adhoc_cmdty_data_map = { } _adhoc_futes_set = { + + # equities 'nq.globex', 'mnq.globex', 'es.globex', 'mes.globex', + + # cypto$ + 'brr.cmecrypto', + 'ethusdrr.cmecrypto', + + # metals + 'xauusd.cmdty', } # exchanges we don't support at the moment due to not knowing From 2471ce446e1c4a3899000693471488f895d98703 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 19 May 2021 08:36:40 -0400 Subject: [PATCH 45/81] Require `.` format to cli --- piker/ui/_chart.py | 4 +++- piker/ui/cli.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 427add22..32743cfb 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1638,8 +1638,10 @@ async def _async_main( ) chart_app.search = search + symbol, _, provider = sym.rpartition('.') + # this internally starts a ``chart_symbol()`` task above - chart_app.load_symbol(brokernames[0], sym, loglevel) + chart_app.load_symbol(provider, symbol, loglevel) # TODO: seems like our incentive for brokerd caching lelel backends = {} diff --git a/piker/ui/cli.py b/piker/ui/cli.py index b674acfe..e65bc379 100644 --- a/piker/ui/cli.py +++ b/piker/ui/cli.py @@ -138,6 +138,13 @@ def chart(config, symbol, profile, pdb): from .. import _profile from ._chart import _main + if '.' not in symbol: + click.echo(click.style( + f'symbol: {symbol} must have a {symbol}. suffix', + fg='red', + )) + return + # toggle to enable profiling _profile._pg_profile = profile From d8a200aadc570853cea096c2e8d42f93cc261c21 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 19 May 2021 08:46:27 -0400 Subject: [PATCH 46/81] Increase completion-tree width, support ctrl-space toggle --- piker/ui/_search.py | 41 ++++++++--------------------------------- 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 96143617..b360f852 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -233,6 +233,9 @@ class CompleterView(QTreeView): # instead its child. self.select_from_idx(model.index(0, 0, QModelIndex())) + # self.resize() + self.show_matches() + def show_matches(self) -> None: self.show() self.resize() @@ -252,7 +255,8 @@ class CompleterView(QTreeView): # TODO: probably make this more general / less hacky self.setMinimumSize(self.width(), rows * row_px) - self.setMaximumSize(self.width(), rows * row_px) + self.setMaximumSize(self.width() + 10, rows * row_px) + self.setFixedWidth(333) def select_previous(self) -> QModelIndex: cidx = self.currentIndex() @@ -328,7 +332,7 @@ class SearchBar(QtWidgets.QLineEdit): # size it as we specify # https://doc.qt.io/qt-5/qsizepolicy.html#Policy-enum self.setSizePolicy( - QtWidgets.QSizePolicy.Fixed, + QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed, ) self.setFont(font.font) @@ -356,7 +360,6 @@ class SearchBar(QtWidgets.QLineEdit): """ psh = super().sizeHint() psh.setHeight(self.dpi_font.px_size + 2) - psh.setWidth(6*6*6) return psh def unfocus(self) -> None: @@ -535,16 +538,12 @@ async def handle_keyboard_input( bar = search.bar view = bar.view view.set_font_size(bar.dpi_font.px_size) - # model = view.model() - # nidx = cidx = view.currentIndex() nidx = view.currentIndex() - # sel = view.selectionModel() symsearch = get_multi_search() send, recv = trio.open_memory_channel(16) async with trio.open_nursery() as n: - # TODO: async debouncing? n.start_soon( partial( fill_results, @@ -557,8 +556,6 @@ async def handle_keyboard_input( async for key, mods, txt in recv_chan: log.debug(f'key: {key}, mods: {mods}, txt: {txt}') - # parent = view.currentIndex() - # cidx = sel.currentIndex() ctrl = False if mods == Qt.ControlModifier: @@ -572,18 +569,6 @@ async def handle_keyboard_input( provider, symbol = value - # # TODO: get rid of this hard coded column -> 1 - # # and use the ``CompleterView`` schema/settings - # # to figure out the desired field(s) - # # https://doc.qt.io/qt-5/qstandarditemmodel.html#itemFromIndex - # node = model.itemFromIndex(cidx.siblingAtColumn(1)) - # if node: - # symbol = node.text() - # provider = node.parent().text() - # # print(f' value: {value}') - # else: - # continue - log.info(f'Requesting symbol: {symbol}.{provider}') # app = search.chart_app @@ -606,7 +591,7 @@ async def handle_keyboard_input( # we're in select mode or cancelling if ctrl: # cancel and close - if key == Qt.Key_C: + if (key == Qt.Key_C) or (key == Qt.Key_Space): search.bar.unfocus() # kill the search and focus back on main chart @@ -634,19 +619,9 @@ async def handle_keyboard_input( if parent_item and parent_item.text() == 'cache': value = search.get_current() + if value is not None: - # continue - provider, symbol = value - # node = model.itemFromIndex( - # i.siblingAtColumn(1) - # ) - # if node: - - # TODO: parse out provider from - # cached value. - # value = node.text() - search.chart_app.load_symbol( provider, symbol, From 64c1d9a9658e859682ba9144727a2ad7cabaa24c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 19 May 2021 08:46:51 -0400 Subject: [PATCH 47/81] Support ctrl-space to open search pane --- piker/ui/_interaction.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 05adfbcd..821a7a3a 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -759,8 +759,9 @@ class ChartView(ViewBox): if mods == QtCore.Qt.AltModifier: pass - # ctlr-l for "lookup" -> open search / lists - if ctrl and key == QtCore.Qt.Key_L: + # ctlr-/ for "lookup", "search" -> open search tree + if ctrl and (key == QtCore.Qt.Key_L or key == QtCore.Qt.Key_Space): + search = self._chart._lc.chart_space.search search.focus() From ddc2c8975ace90f5d6ac3191c553acd7dc227c17 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 19 May 2021 08:47:28 -0400 Subject: [PATCH 48/81] Ignore whitespace patterns in ib search --- piker/brokers/ib.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 289d0583..d4377f7c 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -1286,13 +1286,12 @@ async def open_symbol_search( await trio.sleep(diff) try: pattern = stream.receive_nowait() - # if new: - # pattern = new except trio.WouldBlock: pass - if not pattern: + if not pattern or pattern.isspace(): log.warning('empty pattern received, skipping..') + await stream.send(matches) continue log.debug(f'searching for {pattern}') From 43d73b4a7c56ca9b0e91bf830b70f9e5feec2e0f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 19 May 2021 16:42:26 -0400 Subject: [PATCH 49/81] Info log the current provider search --- piker/ui/_search.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index b360f852..38c646e8 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -383,7 +383,6 @@ async def fill_results( # kb debouncing pauses min_pause_time: float = 0.0616, - # long_pause_time: float = 0.4, max_pause_time: float = 6/16, ) -> None: @@ -674,7 +673,7 @@ def get_multi_search() -> Callable[..., Awaitable]: ) -> None: - log.debug(f'Searching {provider} for "{pattern}"') + log.info(f'Searching {provider} for "{pattern}"') results = await search(pattern) if results: matches[provider] = results From 9d2c8a952675a626cdbbaf571a15433afb8b6089 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 20 May 2021 08:13:50 -0400 Subject: [PATCH 50/81] Factor selection details into completer view methods --- piker/ui/_search.py | 147 +++++++++++++++++++++++++++++--------------- 1 file changed, 97 insertions(+), 50 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 38c646e8..10938468 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -258,37 +258,69 @@ class CompleterView(QTreeView): self.setMaximumSize(self.width() + 10, rows * row_px) self.setFixedWidth(333) - def select_previous(self) -> QModelIndex: - cidx = self.currentIndex() - nidx = self.indexAbove(cidx) - if nidx.parent() is QModelIndex(): - nidx = self.indexAbove(cidx) - breakpoint() + def previous_index(self) -> QModelIndex: - return nidx + cidx = self.selectionModel().currentIndex() + one_above = self.indexAbove(cidx) - def select_next(self) -> QModelIndex: - cidx = self.currentIndex() - nidx = self.indexBelow(cidx) - if nidx.parent() is QModelIndex(): - nidx = self.indexBelow(cidx) - breakpoint() + if one_above.parent() == QModelIndex(): + # if the next node up's parent is the root we don't want to select + # the next node up since it's a top level node and we only + # select entries depth >= 2. - return nidx + # see if one more up is not the root and we can select it. + two_above = self.indexAbove(one_above) + if two_above != QModelIndex(): + return two_above + else: + return cidx + + return one_above # just next up + + def next_index(self) -> QModelIndex: + cidx = self.selectionModel().currentIndex() + one_below = self.indexBelow(cidx) + + if one_below.parent() == QModelIndex(): + # if the next node up's parent is the root we don't want to select + # the next node up since it's a top level node and we only + # select entries depth >= 2. + + # see if one more up is not the root and we can select it. + two_below = self.indexBelow(one_below) + if two_below != QModelIndex(): + return two_below + else: + return cidx + + return one_below # just next up + + # def first_selectable_index(self) -> QModelIndex: + + def select_next(self) -> Tuple[QModelIndex, QStandardItem]: + idx = self.next_index() + assert idx.isValid() + return self.select_from_idx(idx) + + def select_previous(self) -> Tuple[QModelIndex, QStandardItem]: + idx = self.previous_index() + assert idx.isValid() + return self.select_from_idx(idx) def select_from_idx( self, idx: QModelIndex, - ) -> Tuple[QModelIndex, QStandardItem]: + ) -> QStandardItem: + # ) -> Tuple[QModelIndex, QStandardItem]: sel = self.selectionModel() model = self.model() - # select first indented entry - if idx == model.index(0, 0): - idx = self.select_next() + # # select first indented entry + # if idx == model.index(0, 0): + # idx = self.select_next() sel.setCurrentIndex( idx, @@ -296,7 +328,8 @@ class CompleterView(QTreeView): QItemSelectionModel.Rows ) - return idx, model.itemFromIndex(idx) + return model.itemFromIndex(idx) + # return idx, model.itemFromIndex(idx) # def find_matches( # self, @@ -498,7 +531,6 @@ class SearchWidget(QtGui.QWidget): '''Return the current completer tree selection as a tuple ``(parent: str, child: str)`` if valid, else ``None``. - ''' model = self.view.model() sel = self.view.selectionModel() @@ -537,7 +569,7 @@ async def handle_keyboard_input( bar = search.bar view = bar.view view.set_font_size(bar.dpi_font.px_size) - nidx = view.currentIndex() + # nidx = view.currentIndex() symsearch = get_multi_search() send, recv = trio.open_memory_channel(16) @@ -560,6 +592,15 @@ async def handle_keyboard_input( if mods == Qt.ControlModifier: ctrl = True + alt = False + if mods == Qt.AltModifier: + alt = True + + # ctrl + alt as combo + ctlalt = False + if (QtCore.Qt.AltModifier | QtCore.Qt.ControlModifier) == mods: + ctlalt = True + if key in (Qt.Key_Enter, Qt.Key_Return): value = search.get_current() @@ -587,45 +628,51 @@ async def handle_keyboard_input( # - first node index: index = search.index(0, 0, parent) # - root node index: index = search.index(0, 0, QModelIndex()) - # we're in select mode or cancelling - if ctrl: - # cancel and close - if (key == Qt.Key_C) or (key == Qt.Key_Space): - search.bar.unfocus() + # cancel and close + if ctrl and key in { + Qt.Key_C, + Qt.Key_Space, # i feel like this is the "native" one + Qt.Key_Alt, + }: + search.bar.unfocus() - # kill the search and focus back on main chart - if search.chart_app: - search.chart_app.linkedcharts.focus() + # kill the search and focus back on main chart + if search.chart_app: + search.chart_app.linkedcharts.focus() - continue + continue - # result selection nav - if key in (Qt.Key_K, Qt.Key_J): - _search_enabled = False + # selection navigation controls + elif ctrl and key in { - if key == Qt.Key_K: - nidx = view.select_previous() + Qt.Key_K, + Qt.Key_J, - elif key == Qt.Key_J: - nidx = view.select_next() + } or key in { - if nidx.isValid(): - i, item = view.select_from_idx(nidx) + Qt.Key_Up, + Qt.Key_Down, + }: + _search_enabled = False + if key in {Qt.Key_K, Qt.Key_Up}: + item = view.select_previous() - if item: + elif key in {Qt.Key_J, Qt.Key_Down}: + item = view.select_next() - parent_item = item.parent() - if parent_item and parent_item.text() == 'cache': + if item: + parent_item = item.parent() + if parent_item and parent_item.text() == 'cache': - value = search.get_current() + value = search.get_current() - if value is not None: - provider, symbol = value - search.chart_app.load_symbol( - provider, - symbol, - 'info', - ) + if value is not None: + provider, symbol = value + search.chart_app.load_symbol( + provider, + symbol, + 'info', + ) else: # relay to completer task From d5e83e61d44434a13de17791cd4529a1dfc2e584 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 20 May 2021 08:15:51 -0400 Subject: [PATCH 51/81] Reorder, drop some cruft --- piker/ui/_search.py | 178 +++++++++++++++++++------------------------- 1 file changed, 77 insertions(+), 101 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 10938468..4cd12409 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -313,7 +313,6 @@ class CompleterView(QTreeView): idx: QModelIndex, ) -> QStandardItem: - # ) -> Tuple[QModelIndex, QStandardItem]: sel = self.selectionModel() model = self.model() @@ -403,6 +402,76 @@ class SearchBar(QtWidgets.QLineEdit): self.view.hide() +class SearchWidget(QtGui.QWidget): + def __init__( + self, + chart_space: 'ChartSpace', # type: ignore # noqa + columns: List[str] = ['src', 'symbol'], + parent=None, + ): + super().__init__(parent or chart_space) + + # size it as we specify + self.setSizePolicy( + QtWidgets.QSizePolicy.Fixed, + QtWidgets.QSizePolicy.Expanding, + ) + + self.chart_app = chart_space + self.vbox = QtGui.QVBoxLayout(self) + self.vbox.setContentsMargins(0, 0, 0, 0) + self.vbox.setSpacing(4) + + # https://doc.qt.io/qt-5/qlayout.html#SizeConstraint-enum + # self.vbox.setSizeConstraint(QLayout.SetMaximumSize) + + self.view = CompleterView( + parent=self, + labels=columns, + ) + self.bar = SearchBar( + parent=self, + parent_chart=chart_space, + view=self.view, + ) + self.vbox.addWidget(self.bar) + self.vbox.setAlignment(self.bar, Qt.AlignTop | Qt.AlignRight) + self.vbox.addWidget(self.bar.view) + self.vbox.setAlignment(self.view, Qt.AlignTop | Qt.AlignLeft) + + def focus(self) -> None: + # fill cache list + self.view.set_results({'cache': list(self.chart_app._chart_cache)}) + self.bar.focus() + + def get_current(self) -> Optional[Tuple[str, str]]: + '''Return the current completer tree selection as + a tuple ``(parent: str, child: str)`` if valid, else ``None``. + + ''' + model = self.view.model() + sel = self.view.selectionModel() + cidx = sel.currentIndex() + + # TODO: get rid of this hard coded column -> 1 + # and use the ``CompleterView`` schema/settings + # to figure out the desired field(s) + # https://doc.qt.io/qt-5/qstandarditemmodel.html#itemFromIndex + node = model.itemFromIndex(cidx.siblingAtColumn(1)) + if node: + symbol = node.text() + provider = node.parent().text() + + # TODO: move this to somewhere non-search machinery specific? + if provider == 'cache': + symbol, _, provider = symbol.rpartition('.') + + return provider, symbol + + else: + return None + + _search_active: trio.Event = trio.Event() _search_enabled: bool = False @@ -485,76 +554,6 @@ async def fill_results( bar.show() -class SearchWidget(QtGui.QWidget): - def __init__( - self, - chart_space: 'ChartSpace', # type: ignore # noqa - columns: List[str] = ['src', 'symbol'], - parent=None, - ): - super().__init__(parent or chart_space) - - # size it as we specify - self.setSizePolicy( - QtWidgets.QSizePolicy.Fixed, - QtWidgets.QSizePolicy.Expanding, - ) - - self.chart_app = chart_space - self.vbox = QtGui.QVBoxLayout(self) - self.vbox.setContentsMargins(0, 0, 0, 0) - self.vbox.setSpacing(4) - - # https://doc.qt.io/qt-5/qlayout.html#SizeConstraint-enum - # self.vbox.setSizeConstraint(QLayout.SetMaximumSize) - - self.view = CompleterView( - parent=self, - labels=columns, - ) - self.bar = SearchBar( - parent=self, - parent_chart=chart_space, - view=self.view, - ) - self.vbox.addWidget(self.bar) - self.vbox.setAlignment(self.bar, Qt.AlignTop | Qt.AlignRight) - self.vbox.addWidget(self.bar.view) - self.vbox.setAlignment(self.view, Qt.AlignTop | Qt.AlignLeft) - - def focus(self) -> None: - # fill cache list - self.view.set_results({'cache': list(self.chart_app._chart_cache)}) - self.bar.focus() - - def get_current(self) -> Optional[Tuple[str, str]]: - '''Return the current completer tree selection as - a tuple ``(parent: str, child: str)`` if valid, else ``None``. - - ''' - model = self.view.model() - sel = self.view.selectionModel() - cidx = sel.currentIndex() - - # TODO: get rid of this hard coded column -> 1 - # and use the ``CompleterView`` schema/settings - # to figure out the desired field(s) - # https://doc.qt.io/qt-5/qstandarditemmodel.html#itemFromIndex - node = model.itemFromIndex(cidx.siblingAtColumn(1)) - if node: - symbol = node.text() - provider = node.parent().text() - - # TODO: move this to somewhere non-search machinery specific? - if provider == 'cache': - symbol, _, provider = symbol.rpartition('.') - - return provider, symbol - - else: - return None - - async def handle_keyboard_input( search: SearchWidget, @@ -592,14 +591,14 @@ async def handle_keyboard_input( if mods == Qt.ControlModifier: ctrl = True - alt = False - if mods == Qt.AltModifier: - alt = True + # alt = False + # if mods == Qt.AltModifier: + # alt = True - # ctrl + alt as combo - ctlalt = False - if (QtCore.Qt.AltModifier | QtCore.Qt.ControlModifier) == mods: - ctlalt = True + # # ctrl + alt as combo + # ctlalt = False + # if (QtCore.Qt.AltModifier | QtCore.Qt.ControlModifier) == mods: + # ctlalt = True if key in (Qt.Key_Enter, Qt.Key_Return): @@ -762,26 +761,3 @@ async def register_symbol_search( finally: _searcher_cache.pop(provider_name) - - -# if __name__ == '__main__': - - # TODO: simple standalone widget testing script (moreso - # for if/when we decide to expose this module as a standalone - # repo/project). - - # import sys - # local testing of **just** the search UI - # app = QtWidgets.QApplication(sys.argv) - - # syms = [ - # 'XMRUSD', - # 'XBTUSD', - # 'ETHUSD', - # 'XMRXBT', - # 'XDGUSD', - # 'ADAUSD', - # ] - # # search.show() - - # sys.exit(app.exec_()) From 07d8bf14530d0b3816ba26bace7f712f37867f0e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 20 May 2021 10:20:23 -0400 Subject: [PATCH 52/81] Add a `.select_first()` view method + more cleaning --- piker/ui/_search.py | 149 +++++++++++++++++++++----------------------- 1 file changed, 71 insertions(+), 78 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 4cd12409..0688a207 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -178,64 +178,6 @@ class CompleterView(QTreeView): self.setStyleSheet(f"font: {size}px") - def set_results( - self, - results: Dict[str, Sequence[str]], - ) -> None: - - model = self.model() - model.clear() - model.setHorizontalHeaderLabels(self.labels) - - # TODO: wtf.. this model shit - # row_count = model.rowCount() - # if row_count > 0: - # model.removeRows( - # 0, - # row_count, - - # # root index - # model.index(0, 0, QModelIndex()), - # ) - root = model.invisibleRootItem() - - for key, values in results.items(): - - src = QStandardItem(key) - root.appendRow(src) - # self.expand(model.index(1, 0, QModelIndex())) - - # values just needs to be sequence-like - for i, s in enumerate(values): - - ix = QStandardItem(str(i)) - item = QStandardItem(s) - # item.setCheckable(False) - - # Add the item to the model - src.appendRow([ix, item]) - - self.expandAll() - - # XXX: these 2 lines MUST be in sequence in order - # to get the view to show right after typing input. - sel = self.selectionModel() - - # select row without selecting.. :eye_rollzz: - # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex - sel.setCurrentIndex( - model.index(0, 0, QModelIndex()), - QItemSelectionModel.ClearAndSelect | - QItemSelectionModel.Rows - ) - - # ensure we're **not** selecting the first level parent node and - # instead its child. - self.select_from_idx(model.index(0, 0, QModelIndex())) - - # self.resize() - self.show_matches() - def show_matches(self) -> None: self.show() self.resize() @@ -295,32 +237,18 @@ class CompleterView(QTreeView): return one_below # just next up - # def first_selectable_index(self) -> QModelIndex: - - def select_next(self) -> Tuple[QModelIndex, QStandardItem]: - idx = self.next_index() - assert idx.isValid() - return self.select_from_idx(idx) - - def select_previous(self) -> Tuple[QModelIndex, QStandardItem]: - idx = self.previous_index() - assert idx.isValid() - return self.select_from_idx(idx) - def select_from_idx( self, idx: QModelIndex, ) -> QStandardItem: + '''Select and return the item at index ``idx``. + ''' sel = self.selectionModel() model = self.model() - # # select first indented entry - # if idx == model.index(0, 0): - # idx = self.select_next() - sel.setCurrentIndex( idx, QItemSelectionModel.ClearAndSelect | @@ -328,7 +256,72 @@ class CompleterView(QTreeView): ) return model.itemFromIndex(idx) - # return idx, model.itemFromIndex(idx) + + def select_first(self) -> QStandardItem: + '''Select the first depth >= 2 entry from the completer tree and + return it's item. + + ''' + # ensure we're **not** selecting the first level parent node and + # instead its child. + return self.select_from_idx( + self.indexBelow(self.model().index(0, 0, QModelIndex())) + ) + + def select_next(self) -> QStandardItem: + idx = self.next_index() + assert idx.isValid() + return self.select_from_idx(idx) + + def select_previous(self) -> QStandardItem: + idx = self.previous_index() + assert idx.isValid() + return self.select_from_idx(idx) + + def set_results( + self, + results: Dict[str, Sequence[str]], + ) -> None: + + model = self.model() + + # XXX: currently we simply rewrite the model from scratch each call + # since it seems to be super fast anyway. + model.clear() + + model.setHorizontalHeaderLabels(self.labels) + root = model.invisibleRootItem() + + for key, values in results.items(): + + src = QStandardItem(key) + root.appendRow(src) + + # values just needs to be sequence-like + for i, s in enumerate(values): + + ix = QStandardItem(str(i)) + item = QStandardItem(s) + + # Add the item to the model + src.appendRow([ix, item]) + + self.expandAll() + + # XXX: these 2 lines MUST be in sequence in order + # to get the view to show right after typing input. + sel = self.selectionModel() + + # select row without selecting.. :eye_rollzz: + # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex + sel.setCurrentIndex( + model.index(0, 0, QModelIndex()), + QItemSelectionModel.ClearAndSelect | + QItemSelectionModel.Rows + ) + + self.select_first() + self.show_matches() # def find_matches( # self, @@ -444,7 +437,7 @@ class SearchWidget(QtGui.QWidget): self.view.set_results({'cache': list(self.chart_app._chart_cache)}) self.bar.focus() - def get_current(self) -> Optional[Tuple[str, str]]: + def get_current_item(self) -> Optional[Tuple[str, str]]: '''Return the current completer tree selection as a tuple ``(parent: str, child: str)`` if valid, else ``None``. @@ -602,7 +595,7 @@ async def handle_keyboard_input( if key in (Qt.Key_Enter, Qt.Key_Return): - value = search.get_current() + value = search.get_current_item() if value is None: continue @@ -663,7 +656,7 @@ async def handle_keyboard_input( parent_item = item.parent() if parent_item and parent_item.text() == 'cache': - value = search.get_current() + value = search.get_current_item() if value is not None: provider, symbol = value From a4627c2b04b68890598087ea896aaa3e12549cda Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 20 May 2021 10:40:34 -0400 Subject: [PATCH 53/81] Send blank packet on no match to avoid blocking search stream --- piker/brokers/ib.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index d4377f7c..f9f7a8a5 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -179,6 +179,7 @@ _adhoc_futes_set = { _exch_skip_list = { 'ASX', # aussie stocks 'MEXI', # mexican stocks + 'VALUE', # no idea } # https://misc.interactivebrokers.com/cstools/contract_info/v3.10/index.php?action=Conid%20Info&wlId=IB&conid=69067924 @@ -832,7 +833,7 @@ async def get_bars( _err = None - for _ in range(1): + for _ in range(2): try: bars, bars_array = await _trio_run_client_method( @@ -870,6 +871,7 @@ async def get_bars( # and then somehow get that to trigger an event here # that restarts/resumes this task? await tractor.breakpoint() + continue else: # throttle wasn't fixed so error out immediately raise _err @@ -1291,7 +1293,11 @@ async def open_symbol_search( if not pattern or pattern.isspace(): log.warning('empty pattern received, skipping..') - await stream.send(matches) + + # XXX: this unblocks the far end search task which may + # hold up a multi-search nursery block + await stream.send({}) + continue log.debug(f'searching for {pattern}') From 8129fcc6484a7252da9e1d6978c922945d123b77 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 20 May 2021 14:25:23 -0400 Subject: [PATCH 54/81] Ignore key auto-repeats --- piker/ui/_event.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/piker/ui/_event.py b/piker/ui/_event.py index 3d056d43..ac66417f 100644 --- a/piker/ui/_event.py +++ b/piker/ui/_event.py @@ -25,7 +25,7 @@ from PyQt5.QtCore import QEvent import trio -class KeyCloner(QtCore.QObject): +class EventCloner(QtCore.QObject): """Clone and forward keyboard events over a trio memory channel for later async processing. @@ -39,13 +39,21 @@ class KeyCloner(QtCore.QObject): ev: QEvent, ) -> None: - if ev.type() == QEvent.KeyPress: + if ev.type() in { + QEvent.KeyPress, + # QEvent.KeyRelease, + }: + # TODO: is there a global setting for this? + if ev.isAutoRepeat(): + ev.ignore() + return False # XXX: we unpack here because apparently doing it # after pop from the mem chan isn't showing the same # event object? no clue wtf is going on there, likely # something to do with Qt internals and calling the # parent handler? + key = ev.key() mods = ev.modifiers() txt = ev.text() @@ -61,13 +69,14 @@ class KeyCloner(QtCore.QObject): async def open_key_stream( source_widget: QtGui.QWidget, + event_type: QEvent = QEvent.KeyPress, ) -> trio.abc.ReceiveChannel: # 1 to force eager sending send, recv = trio.open_memory_channel(16) - kc = KeyCloner() + kc = EventCloner() kc._send_chan = send source_widget.installEventFilter(kc) From 67498c60af374ffac5e0753300cd4e4241c2e02d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 20 May 2021 14:28:08 -0400 Subject: [PATCH 55/81] More UX features - load previous search state on open - show cached on empty search bar - allow ctrl-u/d to navigate provider "sections" --- piker/ui/_search.py | 68 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 0688a207..b0b47195 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -235,7 +235,7 @@ class CompleterView(QTreeView): else: return cidx - return one_below # just next up + return one_below # just next down def select_from_idx( @@ -278,6 +278,26 @@ class CompleterView(QTreeView): assert idx.isValid() return self.select_from_idx(idx) + def next_section(self, direction: str = 'down') -> QModelIndex: + cidx = start_idx = self.selectionModel().currentIndex() + + # step up levels to depth == 1 + while cidx.parent() != QModelIndex(): + cidx = cidx.parent() + + # move to next section in `direction` + op = {'up': -1, 'down': +1}[direction] + next_row = cidx.row() + op + nidx = self.model().index(next_row, cidx.column(), QModelIndex()) + + # do nothing, if there is no valid "next" section + if not nidx.isValid(): + return self.select_from_idx(start_idx) + + # go to next selectable child item + self.select_from_idx(nidx) + return self.select_next() + def set_results( self, results: Dict[str, Sequence[str]], @@ -370,7 +390,7 @@ class SearchBar(QtWidgets.QLineEdit): # self.setStyleSheet(f"font: 18px") def focus(self) -> None: - self.clear() + self.selectAll() self.show() self.setFocus() @@ -433,8 +453,11 @@ class SearchWidget(QtGui.QWidget): self.vbox.setAlignment(self.view, Qt.AlignTop | Qt.AlignLeft) def focus(self) -> None: - # fill cache list - self.view.set_results({'cache': list(self.chart_app._chart_cache)}) + + if self.view.model().rowCount(QModelIndex()) == 0: + # fill cache list if nothing existing + self.view.set_results({'cache': list(self.chart_app._chart_cache)}) + self.bar.focus() def get_current_item(self) -> Optional[Tuple[str, str]]: @@ -580,15 +603,15 @@ async def handle_keyboard_input( log.debug(f'key: {key}, mods: {mods}, txt: {txt}') - ctrl = False + ctl = False if mods == Qt.ControlModifier: - ctrl = True + ctl = True # alt = False # if mods == Qt.AltModifier: # alt = True - # # ctrl + alt as combo + # # ctl + alt as combo # ctlalt = False # if (QtCore.Qt.AltModifier | QtCore.Qt.ControlModifier) == mods: # ctlalt = True @@ -609,10 +632,21 @@ async def handle_keyboard_input( symbol, 'info', ) + search.bar.clear() + view.set_results({ + 'cache': list(search.chart_app._chart_cache) + }) _search_enabled = False # release kb control of search bar - search.bar.unfocus() + # search.bar.unfocus() + continue + + elif not ctl and not bar.text(): + # if nothing in search text show the cache + view.set_results({ + 'cache': list(search.chart_app._chart_cache) + }) continue # selection tips: @@ -621,7 +655,7 @@ async def handle_keyboard_input( # - root node index: index = search.index(0, 0, QModelIndex()) # cancel and close - if ctrl and key in { + if ctl and key in { Qt.Key_C, Qt.Key_Space, # i feel like this is the "native" one Qt.Key_Alt, @@ -635,7 +669,18 @@ async def handle_keyboard_input( continue # selection navigation controls - elif ctrl and key in { + elif ctl and key in { + Qt.Key_D, + }: + view.next_section(direction='down') + + elif ctl and key in { + Qt.Key_U, + }: + view.next_section(direction='up') + + # selection navigation controls + elif ctl and key in { Qt.Key_K, Qt.Key_J, @@ -654,6 +699,7 @@ async def handle_keyboard_input( if item: parent_item = item.parent() + if parent_item and parent_item.text() == 'cache': value = search.get_current_item() @@ -666,7 +712,7 @@ async def handle_keyboard_input( 'info', ) - else: + elif not ctl: # relay to completer task _search_enabled = True send.send_nowait(search.bar.text()) From 307afb19350984945692dfed62bab4e0465aad7e Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 20 May 2021 14:29:37 -0400 Subject: [PATCH 56/81] Clean some key handling --- piker/ui/_interaction.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/piker/ui/_interaction.py b/piker/ui/_interaction.py index 821a7a3a..9c407f3c 100644 --- a/piker/ui/_interaction.py +++ b/piker/ui/_interaction.py @@ -694,7 +694,7 @@ class ChartView(ViewBox): ev.accept() self.mode.submit_exec() - def keyReleaseEvent(self, ev): + def keyReleaseEvent(self, ev: QtCore.QEvent): """ Key release to normally to trigger release of input mode @@ -713,6 +713,10 @@ class ChartView(ViewBox): # if self.state['mouseMode'] == ViewBox.RectMode: self.setMouseMode(ViewBox.PanMode) + # ctlalt = False + # if (QtCore.Qt.AltModifier | QtCore.Qt.ControlModifier) == mods: + # ctlalt = True + # if self.state['mouseMode'] == ViewBox.RectMode: # if key == QtCore.Qt.Key_Space: if mods == QtCore.Qt.ControlModifier or key == QtCore.Qt.Key_Control: @@ -749,19 +753,20 @@ class ChartView(ViewBox): ctrl = False if mods == QtCore.Qt.ControlModifier: ctrl = True - - if mods == QtCore.Qt.ControlModifier: self.mode._exec_mode = 'live' self._key_active = True - # alt - if mods == QtCore.Qt.AltModifier: - pass + # ctrl + alt + # ctlalt = False + # if (QtCore.Qt.AltModifier | QtCore.Qt.ControlModifier) == mods: + # ctlalt = True # ctlr-/ for "lookup", "search" -> open search tree - if ctrl and (key == QtCore.Qt.Key_L or key == QtCore.Qt.Key_Space): - + if ctrl and key in { + QtCore.Qt.Key_L, + QtCore.Qt.Key_Space, + }: search = self._chart._lc.chart_space.search search.focus() From 212882a5a58fdafef4dae54da61bb96f41a3e486 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 20 May 2021 16:10:42 -0400 Subject: [PATCH 57/81] Don't try to show xhair if no active plot --- piker/ui/_chart.py | 1 + piker/ui/_graphics/_cursor.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 32743cfb..be5aaf7a 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -172,6 +172,7 @@ class ChartSpace(QtGui.QWidget): # XXX: pretty sure we don't need this # remove any existing plots? + # XXX: ahh we might want to support cache unloading.. # self.vbox.removeWidget(self.linkedcharts) # switching to a new viewable chart diff --git a/piker/ui/_graphics/_cursor.py b/piker/ui/_graphics/_cursor.py index 159c773e..0919f6f9 100644 --- a/piker/ui/_graphics/_cursor.py +++ b/piker/ui/_graphics/_cursor.py @@ -435,7 +435,12 @@ class Cursor(pg.GraphicsObject): self, y_label_level: float = None, ) -> None: - g = self.graphics[self.active_plot] + + plot = self.active_plot + if not plot: + return + + g = self.graphics[plot] # show horiz line and y-label g['hl'].show() g['vl'].show() From 0cd3cb33281ebfe8ac4c30e39c22222e6576842a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 20 May 2021 16:11:16 -0400 Subject: [PATCH 58/81] Drop old todo --- piker/clearing/_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/piker/clearing/_client.py b/piker/clearing/_client.py index b643b952..e881a726 100644 --- a/piker/clearing/_client.py +++ b/piker/clearing/_client.py @@ -136,7 +136,6 @@ def get_orders( return _orders -# TODO: make this a ``tractor.msg.pub`` async def send_order_cmds(symbol_key: str): """ Order streaming task: deliver orders transmitted from UI From 82cdb176e12a93c54d543f17b762f8e995af4d0b Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 21 May 2021 12:44:24 -0400 Subject: [PATCH 59/81] Make ctrl-l highlight current text in edit --- piker/ui/_search.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index b0b47195..a57894b0 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -668,6 +668,12 @@ async def handle_keyboard_input( continue + if ctl and key in { + Qt.Key_L, + }: + # like url (link) highlight in a web browser + bar.focus() + # selection navigation controls elif ctl and key in { Qt.Key_D, From 27d704b32ef8331ebf0a21a299515e56bd1942d7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 21 May 2021 12:51:06 -0400 Subject: [PATCH 60/81] To avoid feed breakage, just give up on history after too many throttles for now --- piker/brokers/ib.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index f9f7a8a5..189f800d 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -833,6 +833,7 @@ async def get_bars( _err = None + fails = 0 for _ in range(2): try: @@ -847,7 +848,7 @@ async def get_bars( next_dt = bars[0].date - return bars, bars_array, next_dt + return (bars, bars_array, next_dt), fails except RequestError as err: _err = err @@ -871,10 +872,13 @@ async def get_bars( # and then somehow get that to trigger an event here # that restarts/resumes this task? await tractor.breakpoint() + fails += 1 continue - else: # throttle wasn't fixed so error out immediately - raise _err + return (None, None) + + # else: # throttle wasn't fixed so error out immediately + # raise _err async def backfill_bars( @@ -892,7 +896,7 @@ async def backfill_bars( https://github.com/pikers/piker/issues/128 """ - first_bars, bars_array, next_dt = await get_bars(sym) + (first_bars, bars_array, next_dt), fails = await get_bars(sym) # write historical data to buffer shm.push(bars_array) @@ -904,9 +908,12 @@ async def backfill_bars( i = 0 while i < count: - out = await get_bars(sym, end_dt=next_dt) + out, fails = await get_bars(sym, end_dt=next_dt) - if out is None: + if fails is None or fails > 1: + break + + if out is (None, None): # could be trying to retreive bars over weekend # TODO: add logic here to handle tradable hours and only grab # valid bars in the range From c9cf72d554bc4f09d2cfd442bd524e61eaecfb83 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 23 May 2021 10:53:57 -0400 Subject: [PATCH 61/81] Add remote context allocation api to service daemon This allows for more deterministically managing long running sub-daemon services under `pikerd` using the new context api from `tractor`. The contexts are allocated in an async exit stack and torn down at root daemon termination. Spawn brokerds using this method by changing the persistence entry point to be a `@tractor.context`. --- piker/_daemon.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/piker/_daemon.py b/piker/_daemon.py index 30fdfec2..4c37812d 100644 --- a/piker/_daemon.py +++ b/piker/_daemon.py @@ -18,7 +18,6 @@ Structured, daemon tree service management. """ -from functools import partial from typing import Optional, Union, Callable, Any from contextlib import asynccontextmanager, AsyncExitStack from collections import defaultdict @@ -72,7 +71,7 @@ class Services(BaseModel): ctx, first = await self.ctx_stack.enter_async_context( portal.open_context( target, - **kwargs, + **kwargs, ) ) return ctx From 9bfc230dde7bd481098b7d6f548cb388fbe80c7a Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 24 May 2021 08:31:53 -0400 Subject: [PATCH 62/81] Speedup: load provider searches async at startup --- piker/data/feed.py | 48 +++++++++++---------- piker/ui/_chart.py | 105 +++++++++++++++++++++++++-------------------- 2 files changed, 84 insertions(+), 69 deletions(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index 012df910..1a24b29a 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -148,12 +148,14 @@ async def _setup_persistent_brokerd( async def allocate_persistent_feed( + ctx: tractor.Context, bus: _FeedsBus, brokername: str, symbol: str, loglevel: str, task_status: TaskStatus[trio.CancelScope] = trio.TASK_STATUS_IGNORED, + ) -> None: try: @@ -204,7 +206,6 @@ async def allocate_persistent_feed( bus.feeds[symbol] = (cs, init_msg, first_quote) if opened: - # start history backfill task ``backfill_bars()`` is # a required backend func this must block until shm is # filled with first set of ohlc bars @@ -243,9 +244,9 @@ async def attach_feed_bus( brokername: str, symbol: str, loglevel: str, + ) -> None: - # try: if loglevel is None: loglevel = tractor.current_actor().loglevel @@ -256,15 +257,15 @@ async def attach_feed_bus( assert 'brokerd' in tractor.current_actor().name bus = get_feed_bus(brokername) + sub_only: bool = False + entry = bus.feeds.get(symbol) + + # if no cached feed for this symbol has been created for this + # brokerd yet, start persistent stream and shm writer task in + # service nursery async with bus.task_lock: - task_cs = bus.feeds.get(symbol) - sub_only: bool = False - - # if no cached feed for this symbol has been created for this - # brokerd yet, start persistent stream and shm writer task in - # service nursery - if task_cs is None: + if entry is None: init_msg, first_quote = await bus.nursery.start( partial( allocate_persistent_feed, @@ -276,6 +277,8 @@ async def attach_feed_bus( ) ) bus._subscribers.setdefault(symbol, []).append(ctx) + assert isinstance(bus.feeds[symbol], tuple) + else: sub_only = True @@ -371,9 +374,9 @@ class Feed: # more then one? topics=['local_trades'], ) as self._trade_stream: + yield self._trade_stream else: - yield self._trade_stream @@ -386,9 +389,12 @@ def sym_to_shm_key( @asynccontextmanager async def install_brokerd_search( + portal: tractor._portal.Portal, brokermod: ModuleType, + ) -> None: + async with portal.open_context( brokermod.open_symbol_search ) as (ctx, cache): @@ -402,6 +408,7 @@ async def install_brokerd_search( return await stream.receive() async with _search.register_symbol_search( + provider_name=brokermod.name, search_routine=search, pause_period=brokermod._search_conf.get('pause_period'), @@ -412,18 +419,20 @@ async def install_brokerd_search( @asynccontextmanager async def open_feed( + brokername: str, symbols: Sequence[str], loglevel: Optional[str] = None, + ) -> AsyncIterator[Dict[str, Any]]: - """Open a "data feed" which provides streamed real-time quotes. - - """ + ''' + Open a "data feed" which provides streamed real-time quotes. + ''' sym = symbols[0].lower() # TODO: feed cache locking, right now this is causing - # issues when reconncting to a long running emsd? + # issues when reconnecting to a long running emsd? # global _searcher_cache # async with _cache_lock: @@ -441,6 +450,7 @@ async def open_feed( mod = get_ingestormod(brokername) # no feed for broker exists so maybe spawn a data brokerd + async with maybe_spawn_brokerd( brokername, loglevel=loglevel @@ -497,12 +507,4 @@ async def open_feed( feed._max_sample_rate = max(ohlc_sample_rates) - if brokername in _search._searcher_cache: - yield feed - - else: - async with install_brokerd_search( - feed._brokerd_portal, - mod, - ): - yield feed + yield feed diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index be5aaf7a..749be1d0 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1480,7 +1480,6 @@ async def chart_symbol( f'tick:{symbol.tick_size}' ) - # await tractor.breakpoint() linked_charts = chart_app.linkedcharts linked_charts._symbol = symbol chart = linked_charts.plot_ohlc_main(symbol, bars) @@ -1535,6 +1534,7 @@ async def chart_symbol( }, }) + async with trio.open_nursery() as n: # load initial fsp chain (otherwise known as "indicators") @@ -1558,9 +1558,8 @@ async def chart_symbol( ) # wait for a first quote before we start any update tasks - quote = await feed.receive() - - log.info(f'Received first quote {quote}') + # quote = await feed.receive() + # log.info(f'Received first quote {quote}') n.start_soon( check_for_new_bars, @@ -1581,6 +1580,41 @@ async def chart_symbol( await start_order_mode(chart, symbol, brokername) +async def load_providers( + brokernames: list[str], + loglevel: str, +) -> None: + + # TODO: seems like our incentive for brokerd caching lelel + backends = {} + + async with AsyncExitStack() as stack: + # TODO: spawn these async in nursery. + # load all requested brokerd's at startup and load their + # search engines. + for broker in brokernames: + + # spin up broker daemons for each provider + portal = await stack.enter_async_context( + maybe_spawn_brokerd( + broker, + loglevel=loglevel + ) + ) + + backends[broker] = portal + + await stack.enter_async_context( + feed.install_brokerd_search( + portal, + get_brokermod(broker), + ) + ) + + # keep search engines up until cancelled + await trio.sleep_forever() + + async def _async_main( # implicit required argument provided by ``qtractor_run()`` widgets: Dict[str, Any], @@ -1644,53 +1678,32 @@ async def _async_main( # this internally starts a ``chart_symbol()`` task above chart_app.load_symbol(provider, symbol, loglevel) - # TODO: seems like our incentive for brokerd caching lelel - backends = {} + root_n.start_soon(load_providers, brokernames, loglevel) - async with AsyncExitStack() as stack: + # spin up a search engine for the local cached symbol set + async with _search.register_symbol_search( - # TODO: spawn these async in nursery. + provider_name='cache', + search_routine=partial( + _search.search_simple_dict, + source=chart_app._chart_cache, + ), - # load all requested brokerd's at startup and load their - # search engines. - for broker in brokernames: - portal = await stack.enter_async_context( - maybe_spawn_brokerd( - broker, - loglevel=loglevel - ) + ): + # start handling search bar kb inputs + async with open_key_stream( + search.bar, + ) as key_stream: + + # start kb handling task for searcher + root_n.start_soon( + _search.handle_keyboard_input, + # chart_app, + search, + key_stream, ) - backends[broker] = portal - await stack.enter_async_context( - feed.install_brokerd_search( - portal, - get_brokermod(broker), - ) - ) - - async with _search.register_symbol_search( - - provider_name='cache', - search_routine=partial( - _search.search_simple_dict, - source=chart_app._chart_cache, - ), - - ): - async with open_key_stream( - search.bar, - ) as key_stream: - - # start kb handling task for searcher - root_n.start_soon( - _search.handle_keyboard_input, - # chart_app, - search, - key_stream, - ) - - await trio.sleep_forever() + await trio.sleep_forever() def _main( From af9dcf9230ccf2a095339a02ec324efc84080d24 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 24 May 2021 13:13:22 -0400 Subject: [PATCH 63/81] Use an ordered dict to get LIFO cache sorting on sym selection --- piker/ui/_chart.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 749be1d0..96424c14 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -19,10 +19,11 @@ High level Qt chart widgets. """ import time +from collections import OrderedDict +from contextlib import AsyncExitStack from typing import Tuple, Dict, Any, Optional, Callable from types import ModuleType from functools import partial -from contextlib import AsyncExitStack from PyQt5 import QtCore, QtGui from PyQt5.QtCore import Qt @@ -104,7 +105,7 @@ class ChartSpace(QtGui.QWidget): self.vbox.addLayout(self.toolbar_layout) # self.vbox.addLayout(self.hbox) - self._chart_cache = {} + self._chart_cache = OrderedDict() self.linkedcharts: 'LinkedSplitCharts' = None self.symbol_label: Optional[QtGui.QLabel] = None @@ -116,6 +117,8 @@ class ChartSpace(QtGui.QWidget): linked_charts: 'LinkedSplitCharts', # type: ignore ) -> None: self._chart_cache[symbol_key] = linked_charts + # re-sort list in LIFO order + self._chart_cache.move_to_end(symbol_key, last=False) def get_chart_symbol( self, @@ -191,7 +194,8 @@ class ChartSpace(QtGui.QWidget): ) self.vbox.addWidget(linkedcharts) - self.set_chart_symbol(fqsn, linkedcharts) + + self.set_chart_symbol(fqsn, linkedcharts) # chart is already in memory so just focus it if self.linkedcharts: From 59475cfd81b3c9c9b02824d7bb726922e338b82d Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 25 May 2021 06:17:18 -0400 Subject: [PATCH 64/81] Store lowercase symbols within piker data internals --- piker/data/_sampling.py | 3 ++- piker/data/feed.py | 12 ++++++++++-- piker/ui/_chart.py | 2 +- piker/ui/_search.py | 1 - 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/piker/data/_sampling.py b/piker/data/_sampling.py index 2079eb71..566f2b07 100644 --- a/piker/data/_sampling.py +++ b/piker/data/_sampling.py @@ -227,7 +227,8 @@ async def sample_and_broadcast( # end up triggering backpressure which which will # eventually block this producer end of the feed and # thus other consumers still attached. - subs = bus._subscribers[sym] + subs = bus._subscribers[sym.lower()] + for ctx in subs: # print(f'sub is {ctx.chan.uid}') try: diff --git a/piker/data/feed.py b/piker/data/feed.py index 1a24b29a..1097de50 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -203,7 +203,10 @@ async def allocate_persistent_feed( # TODO: make this into a composed type which also # contains the backfiller cs for individual super-based # resspawns when needed. - bus.feeds[symbol] = (cs, init_msg, first_quote) + + # XXX: the ``symbol`` here is put into our native piker format (i.e. + # lower case). + bus.feeds[symbol.lower()] = (cs, init_msg, first_quote) if opened: # start history backfill task ``backfill_bars()`` is @@ -272,7 +275,12 @@ async def attach_feed_bus( ctx=ctx, bus=bus, brokername=brokername, + + # here we pass through the selected symbol in native + # "format" (i.e. upper vs. lowercase depending on + # provider). symbol=symbol, + loglevel=loglevel, ) ) @@ -411,7 +419,7 @@ async def install_brokerd_search( provider_name=brokermod.name, search_routine=search, - pause_period=brokermod._search_conf.get('pause_period'), + pause_period=brokermod._search_conf.get('pause_period', 0.0616), ): yield diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 96424c14..64646da3 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -1538,7 +1538,6 @@ async def chart_symbol( }, }) - async with trio.open_nursery() as n: # load initial fsp chain (otherwise known as "indicators") @@ -1598,6 +1597,7 @@ async def load_providers( # search engines. for broker in brokernames: + log.info(f'Loading brokerd for {broker}') # spin up broker daemons for each provider portal = await stack.enter_async_context( maybe_spawn_brokerd( diff --git a/piker/ui/_search.py b/piker/ui/_search.py index a57894b0..27b3e360 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -497,7 +497,6 @@ async def fill_results( search: SearchBar, symsearch: Callable[..., Awaitable], recv_chan: trio.abc.ReceiveChannel, - # cached_symbols: Dict[str, # kb debouncing pauses min_pause_time: float = 0.0616, From 44f4fdf04309cbd4175c53fa46e37cb291f14ee7 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 25 May 2021 06:33:07 -0400 Subject: [PATCH 65/81] Type annot the internal symbol cache --- piker/brokers/kraken.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index 62fc6115..f2adccf4 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -153,7 +153,7 @@ class Client: 'User-Agent': 'krakenex/2.1.0 (+https://github.com/veox/python3-krakenex)' }) - self._pairs = None + self._pairs: list[str] = [] @property def pairs(self) -> Dict[str, Any]: @@ -289,7 +289,7 @@ class Client: async def get_client() -> Client: client = Client() - # load all symbols locally for fast search + # at startup, load all symbols locally for fast search await client.cache_symbols() yield client From 46d88965d3aa4497005113b9d641bf473f367c42 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 25 May 2021 11:55:24 -0400 Subject: [PATCH 66/81] Get LIFO sort on cache syms working properly --- piker/ui/_chart.py | 18 ++++++++++-------- piker/ui/_search.py | 25 +++++++++++++++++-------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index 64646da3..c79c001c 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -19,7 +19,6 @@ High level Qt chart widgets. """ import time -from collections import OrderedDict from contextlib import AsyncExitStack from typing import Tuple, Dict, Any, Optional, Callable from types import ModuleType @@ -105,7 +104,7 @@ class ChartSpace(QtGui.QWidget): self.vbox.addLayout(self.toolbar_layout) # self.vbox.addLayout(self.hbox) - self._chart_cache = OrderedDict() + self._chart_cache = {} self.linkedcharts: 'LinkedSplitCharts' = None self.symbol_label: Optional[QtGui.QLabel] = None @@ -115,10 +114,12 @@ class ChartSpace(QtGui.QWidget): self, symbol_key: str, # of form . linked_charts: 'LinkedSplitCharts', # type: ignore + ) -> None: - self._chart_cache[symbol_key] = linked_charts - # re-sort list in LIFO order - self._chart_cache.move_to_end(symbol_key, last=False) + # re-sort org cache symbol list in LIFO order + cache = self._chart_cache + cache.pop(symbol_key, None) + cache[symbol_key] = linked_charts def get_chart_symbol( self, @@ -176,7 +177,7 @@ class ChartSpace(QtGui.QWidget): # XXX: pretty sure we don't need this # remove any existing plots? # XXX: ahh we might want to support cache unloading.. - # self.vbox.removeWidget(self.linkedcharts) + self.vbox.removeWidget(self.linkedcharts) # switching to a new viewable chart if linkedcharts is None or reset: @@ -193,9 +194,10 @@ class ChartSpace(QtGui.QWidget): loglevel, ) - self.vbox.addWidget(linkedcharts) - self.set_chart_symbol(fqsn, linkedcharts) + self.set_chart_symbol(fqsn, linkedcharts) + + self.vbox.addWidget(linkedcharts) # chart is already in memory so just focus it if self.linkedcharts: diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 27b3e360..1949cbd2 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -456,7 +456,7 @@ class SearchWidget(QtGui.QWidget): if self.view.model().rowCount(QModelIndex()) == 0: # fill cache list if nothing existing - self.view.set_results({'cache': list(self.chart_app._chart_cache)}) + self.view.set_results({'cache': list(reversed(self.chart_app._chart_cache))}) self.bar.focus() @@ -625,15 +625,24 @@ async def handle_keyboard_input( log.info(f'Requesting symbol: {symbol}.{provider}') - # app = search.chart_app - search.chart_app.load_symbol( + chart = search.chart_app + chart.load_symbol( provider, symbol, 'info', ) + + # fully qualified symbol name (SNS i guess is what we're making?) + fqsn = '.'.join([symbol, provider]).lower() + + # Re-order the symbol cache on the chart to display in + # LIFO order. this is normally only done internally by + # the chart on new symbols being loaded into memory + chart.set_chart_symbol(fqsn, chart.linkedcharts) + search.bar.clear() view.set_results({ - 'cache': list(search.chart_app._chart_cache) + 'cache': list(reversed(chart._chart_cache)) }) _search_enabled = False @@ -644,7 +653,7 @@ async def handle_keyboard_input( elif not ctl and not bar.text(): # if nothing in search text show the cache view.set_results({ - 'cache': list(search.chart_app._chart_cache) + 'cache': list(reversed(chart._chart_cache)) }) continue @@ -662,8 +671,8 @@ async def handle_keyboard_input( search.bar.unfocus() # kill the search and focus back on main chart - if search.chart_app: - search.chart_app.linkedcharts.focus() + if chart: + chart.linkedcharts.focus() continue @@ -711,7 +720,7 @@ async def handle_keyboard_input( if value is not None: provider, symbol = value - search.chart_app.load_symbol( + chart.load_symbol( provider, symbol, 'info', From 924960a35967b0e81a0880bec058b9764ff4e907 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 25 May 2021 17:10:46 -0400 Subject: [PATCH 67/81] Add label to search bar --- piker/ui/_chart.py | 8 +------- piker/ui/_search.py | 37 ++++++++++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index c79c001c..d3661922 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -106,8 +106,6 @@ class ChartSpace(QtGui.QWidget): self._chart_cache = {} self.linkedcharts: 'LinkedSplitCharts' = None - self.symbol_label: Optional[QtGui.QLabel] = None - self._root_n: Optional[trio.Nursery] = None def set_chart_symbol( @@ -1660,11 +1658,7 @@ async def _async_main( chart_app._root_n = root_n # setup search widget - # search.installEventFilter(self) - - search = _search.SearchWidget( - chart_space=chart_app, - ) + search = _search.SearchWidget(chart_space=chart_app) # the main chart's view is given focus at startup search.bar.unfocus() diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 1949cbd2..68c01999 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -408,7 +408,7 @@ class SearchBar(QtWidgets.QLineEdit): return psh def unfocus(self) -> None: - self.hide() + self.parent().hide() self.clearFocus() if self.view: @@ -431,10 +431,29 @@ class SearchWidget(QtGui.QWidget): ) self.chart_app = chart_space + self.vbox = QtGui.QVBoxLayout(self) self.vbox.setContentsMargins(0, 0, 0, 0) self.vbox.setSpacing(4) + # split layout for the (label:| search bar entry) + self.bar_hbox = QtGui.QHBoxLayout(self) + self.bar_hbox.setContentsMargins(0, 0, 0, 0) + self.bar_hbox.setSpacing(4) + + self.label = label = QtGui.QLabel(parent=self) + label.setTextFormat(3) # markdown + label.setFont(_font.font) + label.setMargin(4) + label.setText("`search`:") + label.show() + label.setAlignment( + QtCore.Qt.AlignVCenter + | QtCore.Qt.AlignLeft + ) + + self.bar_hbox.addWidget(label) + # https://doc.qt.io/qt-5/qlayout.html#SizeConstraint-enum # self.vbox.setSizeConstraint(QLayout.SetMaximumSize) @@ -447,7 +466,12 @@ class SearchWidget(QtGui.QWidget): parent_chart=chart_space, view=self.view, ) - self.vbox.addWidget(self.bar) + self.bar_hbox.addWidget(self.bar) + + # self.vbox.addWidget(self.bar) + # self.vbox.setAlignment(self.bar, Qt.AlignTop | Qt.AlignRight) + self.vbox.addLayout(self.bar_hbox) + self.vbox.setAlignment(self.bar, Qt.AlignTop | Qt.AlignRight) self.vbox.addWidget(self.bar.view) self.vbox.setAlignment(self.view, Qt.AlignTop | Qt.AlignLeft) @@ -456,9 +480,11 @@ class SearchWidget(QtGui.QWidget): if self.view.model().rowCount(QModelIndex()) == 0: # fill cache list if nothing existing - self.view.set_results({'cache': list(reversed(self.chart_app._chart_cache))}) + self.view.set_results( + {'cache': list(reversed(self.chart_app._chart_cache))}) self.bar.focus() + self.show() def get_current_item(self) -> Optional[Tuple[str, str]]: '''Return the current completer tree selection as @@ -580,6 +606,7 @@ async def handle_keyboard_input( global _search_active, _search_enabled # startup + chart = search.chart_app bar = search.bar view = bar.view view.set_font_size(bar.dpi_font.px_size) @@ -625,14 +652,14 @@ async def handle_keyboard_input( log.info(f'Requesting symbol: {symbol}.{provider}') - chart = search.chart_app chart.load_symbol( provider, symbol, 'info', ) - # fully qualified symbol name (SNS i guess is what we're making?) + # fully qualified symbol name (SNS i guess is what we're + # making?) fqsn = '.'.join([symbol, provider]).lower() # Re-order the symbol cache on the chart to display in From c478ddaed0a82d9fef4ea73b717a0ed527a5ac87 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 26 May 2021 12:46:24 -0400 Subject: [PATCH 68/81] Disable cursor blink globally --- piker/ui/_exec.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/piker/ui/_exec.py b/piker/ui/_exec.py index 96755fab..85488df8 100644 --- a/piker/ui/_exec.py +++ b/piker/ui/_exec.py @@ -145,6 +145,9 @@ def run_qtractor( # currently seem tricky.. app.setQuitOnLastWindowClosed(False) + # XXX: lmfao, this is how you disable text edit cursor blinking..smh + app.setCursorFlashTime(0) + # set global app singleton global _qt_app _qt_app = app From 89beb928668d3d2fbe87a8fea5084cb15b0316b8 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 26 May 2021 13:49:14 -0400 Subject: [PATCH 69/81] Add api for per-section filling/clearing Makes it so we can move toward separate provider results fills in an async way, on demand. Also, - add depth 1 iteration helper method - add section finder helper method - fix last selection loading to be mostly consistent --- piker/ui/_search.py | 255 ++++++++++++++++++++++++++++---------------- 1 file changed, 163 insertions(+), 92 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 68c01999..1454a6d6 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -31,6 +31,7 @@ qompleterz: embeddable search and complete using trio, Qt and fuzzywuzzy. # https://github.com/qutebrowser/qutebrowser/blob/master/qutebrowser/completion/completiondelegate.py#L243 # https://forum.qt.io/topic/61343/highlight-matched-substrings-in-qstyleditemdelegate +from collections import defaultdict from contextlib import asynccontextmanager from functools import partial from typing import ( @@ -91,18 +92,6 @@ class SimpleDelegate(QStyledItemDelegate): super().__init__(parent) self.dpi_font = font - # def sizeHint(self, *args) -> QtCore.QSize: - # """ - # Scale edit box to size of dpi aware font. - - # """ - # psh = super().sizeHint(*args) - # # psh.setHeight(self.dpi_font.px_size + 2) - - # psh.setHeight(18) - # # psh.setHeight(18) - # return psh - class CompleterView(QTreeView): @@ -141,6 +130,12 @@ class CompleterView(QTreeView): self.setAnimated(False) self.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) + # TODO: this up front? + # self.setSelectionModel( + # QItemSelectionModel.ClearAndSelect | + # QItemSelectionModel.Rows + # ) + # self.setVerticalBarPolicy(Qt.ScrollBarAlwaysOff) # self.setSizeAdjustPolicy(QAbstractScrollArea.AdjustIgnored) @@ -148,7 +143,6 @@ class CompleterView(QTreeView): model.setHorizontalHeaderLabels(labels) self._font_size: int = 0 # pixels - # self._cache: Dict[str, List[str]] = {} # def viewportSizeHint(self) -> QtCore.QSize: # vps = super().viewportSizeHint() @@ -178,10 +172,6 @@ class CompleterView(QTreeView): self.setStyleSheet(f"font: {size}px") - def show_matches(self) -> None: - self.show() - self.resize() - def resize(self): model = self.model() cols = model.columnCount() @@ -200,6 +190,10 @@ class CompleterView(QTreeView): self.setMaximumSize(self.width() + 10, rows * row_px) self.setFixedWidth(333) + def is_selecting_d1(self) -> bool: + cidx = self.selectionModel().currentIndex() + return cidx.parent() == QModelIndex() + def previous_index(self) -> QModelIndex: cidx = self.selectionModel().currentIndex() @@ -264,9 +258,12 @@ class CompleterView(QTreeView): ''' # ensure we're **not** selecting the first level parent node and # instead its child. - return self.select_from_idx( - self.indexBelow(self.model().index(0, 0, QModelIndex())) - ) + model = self.model() + for idx, item in self.iter_d1(): + if model.rowCount(idx) == 0: + continue + else: + return self.select_from_idx(self.indexBelow(idx)) def select_next(self) -> QStandardItem: idx = self.next_index() @@ -298,62 +295,113 @@ class CompleterView(QTreeView): self.select_from_idx(nidx) return self.select_next() - def set_results( + def iter_d1( self, - results: Dict[str, Sequence[str]], - ) -> None: + ) -> tuple[QModelIndex, QStandardItem]: model = self.model() + isections = model.rowCount() - # XXX: currently we simply rewrite the model from scratch each call - # since it seems to be super fast anyway. - model.clear() + # much thanks to following code to figure out breadth-first + # traversing from the root node: + # https://stackoverflow.com/a/33126689 + for i in range(isections): + idx = model.index(i, 0, QModelIndex()) + item = model.itemFromIndex(idx) + yield idx, item + + def find_section( + self, + section: str, + + ) -> Optional[QModelIndex]: + '''Find the *first* depth = 1 section matching ``section`` in + the tree and return its index. + + ''' + for idx, item in self.iter_d1(): + if item.text() == section: + return idx + else: + # caller must expect his + return None + + def clear_section( + self, + section: str, + status_field: str = None, + + ) -> None: + '''Clear all result-rows from under the depth = 1 section. + + ''' + idx = self.find_section(section) + model = self.model() + + if idx is not None: + if model.hasChildren(idx): + rows = model.rowCount(idx) + # print(f'removing {rows} from {section}') + assert model.removeRows(0, rows, parent=idx) + + # remove section as well + # model.removeRow(i, QModelIndex()) + + return idx + else: + return None + + def set_section_entries( + self, + section: str, + values: Sequence[str], + clear_all: bool = False, + + ) -> None: + '''Set result-rows for depth = 1 tree section ``section``. + + ''' + model = self.model() + if clear_all: + # XXX: rewrite the model from scratch if caller requests it + model.clear() model.setHorizontalHeaderLabels(self.labels) - root = model.invisibleRootItem() - for key, values in results.items(): - src = QStandardItem(key) - root.appendRow(src) + section_idx = self.clear_section(section) - # values just needs to be sequence-like - for i, s in enumerate(values): + # for key, values in results.items(): - ix = QStandardItem(str(i)) - item = QStandardItem(s) + if section_idx is None: + root = model.invisibleRootItem() + section_item = QStandardItem(section) + root.appendRow(section_item) + else: + section_item = model.itemFromIndex(section_idx) - # Add the item to the model - src.appendRow([ix, item]) + # values just needs to be sequence-like + for i, s in enumerate(values): + + ix = QStandardItem(str(i)) + item = QStandardItem(s) + + # Add the item to the model + section_item.appendRow([ix, item]) self.expandAll() - # XXX: these 2 lines MUST be in sequence in order - # to get the view to show right after typing input. - sel = self.selectionModel() - - # select row without selecting.. :eye_rollzz: - # https://doc.qt.io/qt-5/qabstractitemview.html#setCurrentIndex - sel.setCurrentIndex( - model.index(0, 0, QModelIndex()), - QItemSelectionModel.ClearAndSelect | - QItemSelectionModel.Rows - ) - + # XXX: THE BELOW LINE MUST BE CALLED. + # this stuff is super finicky and if not done right will cause + # Qt crashes out our buttz. it's required in order to get the + # view to show right after typing input. self.select_first() + self.show_matches() - # def find_matches( - # self, - # field: str, - # txt: str, - # ) -> List[QStandardItem]: - # model = self.model() - # items = model.findItems( - # txt, - # Qt.MatchContains, - # self.field_to_col(field), - # ) + def show_matches(self) -> None: + self.show() + self.resize() class SearchBar(QtWidgets.QLineEdit): @@ -480,8 +528,11 @@ class SearchWidget(QtGui.QWidget): if self.view.model().rowCount(QModelIndex()) == 0: # fill cache list if nothing existing - self.view.set_results( - {'cache': list(reversed(self.chart_app._chart_cache))}) + self.view.set_section_entries( + 'cache', + list(reversed(self.chart_app._chart_cache)), + clear_all=True, + ) self.bar.focus() self.show() @@ -521,7 +572,7 @@ _search_enabled: bool = False async def fill_results( search: SearchBar, - symsearch: Callable[..., Awaitable], + # multisearch: Callable[..., Awaitable], recv_chan: trio.abc.ReceiveChannel, # kb debouncing pauses @@ -535,11 +586,15 @@ async def fill_results( """ global _search_active, _search_enabled + multisearch = get_multi_search() + bar = search.bar view = bar.view + view.select_from_idx(QModelIndex()) last_text = bar.text() repeats = 0 + last_patt = None while True: await _search_active.wait() @@ -584,15 +639,36 @@ async def fill_results( log.debug(f'Search req for {text}') - results = await symsearch(text, period=period) + # issue multi-provider fan-out search request + results = await multisearch(text, period=period) - log.debug(f'Received search result {results}') + # matches = {} + # unmatches = [] - if results and _search_enabled: + if _search_enabled: - # show the results in the completer view - view.set_results(results) - bar.show() + for (provider, pattern), output in results.items(): + if output: + # matches[provider] = output + view.set_section_entries( + section=provider, + values=output, + ) + + else: + view.clear_section(provider) + + if last_patt is None or last_patt != text: + view.select_first() + + # only change select on first search iteration, + # late results from other providers should **not** + # move the current selection + # if pattern not in patt_searched: + # patt_searched[pattern].append(provider) + + last_patt = text + bar.show() async def handle_keyboard_input( @@ -610,17 +686,19 @@ async def handle_keyboard_input( bar = search.bar view = bar.view view.set_font_size(bar.dpi_font.px_size) - # nidx = view.currentIndex() - symsearch = get_multi_search() send, recv = trio.open_memory_channel(16) async with trio.open_nursery() as n: + + # start a background multi-searcher task which receives + # patterns relayed from this keyboard input handler and + # async updates the completer view's results. n.start_soon( partial( fill_results, search, - symsearch, + # multisearch, recv, ) ) @@ -633,10 +711,6 @@ async def handle_keyboard_input( if mods == Qt.ControlModifier: ctl = True - # alt = False - # if mods == Qt.AltModifier: - # alt = True - # # ctl + alt as combo # ctlalt = False # if (QtCore.Qt.AltModifier | QtCore.Qt.ControlModifier) == mods: @@ -668,27 +742,24 @@ async def handle_keyboard_input( chart.set_chart_symbol(fqsn, chart.linkedcharts) search.bar.clear() - view.set_results({ - 'cache': list(reversed(chart._chart_cache)) - }) + view.set_section_entries( + 'cache', + values=list(reversed(chart._chart_cache)), + clear_all=True, + ) _search_enabled = False - # release kb control of search bar - # search.bar.unfocus() continue elif not ctl and not bar.text(): # if nothing in search text show the cache - view.set_results({ - 'cache': list(reversed(chart._chart_cache)) - }) + view.set_section_entries( + 'cache', + list(reversed(chart._chart_cache)), + clear_all=True, + ) continue - # selection tips: - # - get parent node: search.index(row, 0) - # - first node index: index = search.index(0, 0, parent) - # - root node index: index = search.index(0, 0, QModelIndex()) - # cancel and close if ctl and key in { Qt.Key_C, @@ -789,7 +860,7 @@ def get_multi_search() -> Callable[..., Awaitable]: period: str, ) -> dict: - + # nonlocal matches matches = {} async def pack_matches( @@ -801,8 +872,8 @@ def get_multi_search() -> Callable[..., Awaitable]: log.info(f'Searching {provider} for "{pattern}"') results = await search(pattern) - if results: - matches[provider] = results + # print(f'results from {provider}: {results}') + matches[(provider, pattern)] = results # TODO: make this an async stream? async with trio.open_nursery() as n: @@ -811,7 +882,7 @@ def get_multi_search() -> Callable[..., Awaitable]: # only conduct search on this backend if it's registered # for the corresponding pause period. - if period >= min_pause: + if period >= min_pause and (provider, pattern) not in matches: # print( # f'searching {provider} after {period} > {min_pause}') n.start_soon(pack_matches, provider, pattern, search) From 607e1a82995af5280fad9bfc51759b61d742b2d9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 26 May 2021 16:50:42 -0400 Subject: [PATCH 70/81] Add per-provider-async searching with status updates --- piker/ui/_search.py | 180 ++++++++++++++++++++++++++------------------ 1 file changed, 107 insertions(+), 73 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 1454a6d6..5adb23cf 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -44,6 +44,7 @@ import time from fuzzywuzzy import process as fuzzy import trio +from trio_typing import TaskStatus from PyQt5 import QtCore, QtGui from PyQt5 import QtWidgets from PyQt5.QtCore import ( @@ -347,6 +348,21 @@ class CompleterView(QTreeView): # remove section as well # model.removeRow(i, QModelIndex()) + if status_field is not None: + model.setItem(idx.row(), 1, QStandardItem(status_field)) + else: + model.setItem(idx.row(), 1, QStandardItem()) + + # XXX: not idea how to use this + # model.setItemData( + # idx, + # { + # 0: 'cache', + # 1: 'searching', + # } + # ) + self.resize() + return idx else: return None @@ -368,15 +384,15 @@ class CompleterView(QTreeView): model.setHorizontalHeaderLabels(self.labels) - section_idx = self.clear_section(section) - # for key, values in results.items(): - + # if we can't find a section start adding to the root if section_idx is None: root = model.invisibleRootItem() section_item = QStandardItem(section) - root.appendRow(section_item) + blank = QStandardItem('') + root.appendRow([section_item, blank]) + else: section_item = model.itemFromIndex(section_idx) @@ -553,7 +569,11 @@ class SearchWidget(QtGui.QWidget): node = model.itemFromIndex(cidx.siblingAtColumn(1)) if node: symbol = node.text() - provider = node.parent().text() + try: + provider = node.parent().text() + except AttributeError: + # no text set + return None # TODO: move this to somewhere non-search machinery specific? if provider == 'cache': @@ -569,6 +589,47 @@ _search_active: trio.Event = trio.Event() _search_enabled: bool = False +async def pack_matches( + view: CompleterView, + has_results: dict[str, set[str]], + matches: dict[(str, str), [str]], + provider: str, + pattern: str, + search: Callable[..., Awaitable[dict]], + task_status: TaskStatus[ + trio.CancelScope] = trio.TASK_STATUS_IGNORED, + +) -> None: + + log.info(f'Searching {provider} for "{pattern}"') + if provider != 'cache': + view.set_section_entries( + section=provider, + values=[], + ) + view.clear_section(provider, status_field='-> searchin..') + + else: # for the cache just clear it's entries and don't put a status + view.clear_section(provider) + + with trio.CancelScope() as cs: + task_status.started(cs) + # ensure ^ status is updated + results = await search(pattern) + + if provider != 'cache': + matches[(provider, pattern)] = results + + # print(f'results from {provider}: {results}') + has_results[pattern].add(provider) + + if results: + view.set_section_entries( + section=provider, + values=results, + ) + + async def fill_results( search: SearchBar, @@ -584,9 +645,7 @@ async def fill_results( completion results. """ - global _search_active, _search_enabled - - multisearch = get_multi_search() + global _search_active, _search_enabled, _searcher_cache bar = search.bar view = bar.view @@ -596,6 +655,10 @@ async def fill_results( repeats = 0 last_patt = None + # cache of prior patterns to search results + matches = defaultdict(list) + has_results: defaultdict[str, set[str]] = defaultdict(set) + while True: await _search_active.wait() period = None @@ -639,33 +702,47 @@ async def fill_results( log.debug(f'Search req for {text}') - # issue multi-provider fan-out search request - results = await multisearch(text, period=period) + already_has_results = has_results[text] - # matches = {} - # unmatches = [] + # issue multi-provider fan-out search request and place + # "searching.." statuses on outstanding results providers + async with trio.open_nursery() as n: - if _search_enabled: + for provider, (search, pause) in _searcher_cache.items(): + print(provider) - for (provider, pattern), output in results.items(): - if output: - # matches[provider] = output - view.set_section_entries( - section=provider, - values=output, + # TODO: put "searching..." status in result field + + if provider != 'cache': + view.clear_section( + provider, status_field='-> searchin..') + + # only conduct search on this backend if it's + # registered for the corresponding pause period. + if (period >= pause) and ( + provider not in already_has_results + ): + await n.start( + pack_matches, + view, + has_results, + matches, + provider, + text, + search ) + else: # already has results for this input text + results = matches[(provider, text)] + if results: + view.set_section_entries( + section=provider, + values=results, + ) + else: + view.clear_section(provider) - else: - view.clear_section(provider) - - if last_patt is None or last_patt != text: - view.select_first() - - # only change select on first search iteration, - # late results from other providers should **not** - # move the current selection - # if pattern not in patt_searched: - # patt_searched[pattern].append(provider) + if last_patt is None or last_patt != text: + view.select_first() last_patt = text bar.show() @@ -698,7 +775,6 @@ async def handle_keyboard_input( partial( fill_results, search, - # multisearch, recv, ) ) @@ -815,7 +891,6 @@ async def handle_keyboard_input( if parent_item and parent_item.text() == 'cache': value = search.get_current_item() - if value is not None: provider, symbol = value chart.load_symbol( @@ -851,47 +926,6 @@ async def search_simple_dict( _searcher_cache: Dict[str, Callable[..., Awaitable]] = {} -def get_multi_search() -> Callable[..., Awaitable]: - - global _searcher_cache - - async def multisearcher( - pattern: str, - period: str, - - ) -> dict: - # nonlocal matches - matches = {} - - async def pack_matches( - provider: str, - pattern: str, - search: Callable[..., Awaitable[dict]], - - ) -> None: - - log.info(f'Searching {provider} for "{pattern}"') - results = await search(pattern) - # print(f'results from {provider}: {results}') - matches[(provider, pattern)] = results - - # TODO: make this an async stream? - async with trio.open_nursery() as n: - - for provider, (search, min_pause) in _searcher_cache.items(): - - # only conduct search on this backend if it's registered - # for the corresponding pause period. - if period >= min_pause and (provider, pattern) not in matches: - # print( - # f'searching {provider} after {period} > {min_pause}') - n.start_soon(pack_matches, provider, pattern, search) - - return matches - - return multisearcher - - @asynccontextmanager async def register_symbol_search( From ab3adcee9e2976dc98bce642d5ad6c0315ba0566 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Wed, 26 May 2021 23:47:20 -0400 Subject: [PATCH 71/81] Get basic switch-on-click mouse support working --- piker/ui/_search.py | 45 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 5adb23cf..9b698ef3 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -122,6 +122,8 @@ class CompleterView(QTreeView): # TODO: size this based on DPI font self.setIndentation(20) + self.pressed.connect(self.on_pressed) + # self.setUniformRowHeights(True) # self.setColumnWidth(0, 3) @@ -145,6 +147,32 @@ class CompleterView(QTreeView): self._font_size: int = 0 # pixels + def on_pressed(self, idx: QModelIndex) -> None: + + search = self.parent() + value = search.get_current_item() + + if value is not None: + provider, symbol = value + chart = search.chart_app + + chart.load_symbol( + provider, + symbol, + 'info', + ) + + # fully qualified symbol name (SNS i guess is what we're + # making?) + fqsn = '.'.join([symbol, provider]).lower() + + # Re-order the symbol cache on the chart to display in + # LIFO order. this is normally only done internally by + # the chart on new symbols being loaded into memory + chart.set_chart_symbol(fqsn, chart.linkedcharts) + + search.focus() + # def viewportSizeHint(self) -> QtCore.QSize: # vps = super().viewportSizeHint() # return QSize(vps.width(), _font.px_size * 6 * 2) @@ -165,7 +193,7 @@ class CompleterView(QTreeView): def set_font_size(self, size: int = 18): # dpi_px_size = _font.px_size - print(size) + # print(size) if size < 0: size = 16 @@ -681,10 +709,10 @@ async def fill_results( continue text = bar.text() - print(f'search: {text}') + # print(f'search: {text}') if not text: - print('idling') + # print('idling') _search_active = trio.Event() break @@ -697,7 +725,7 @@ async def fill_results( repeats += 1 if not _search_enabled: - print('search currently disabled') + # print('search currently disabled') break log.debug(f'Search req for {text}') @@ -708,10 +736,7 @@ async def fill_results( # "searching.." statuses on outstanding results providers async with trio.open_nursery() as n: - for provider, (search, pause) in _searcher_cache.items(): - print(provider) - - # TODO: put "searching..." status in result field + for provider, (search, pause) in _searcher_cache.copy().items(): if provider != 'cache': view.clear_section( @@ -741,8 +766,8 @@ async def fill_results( else: view.clear_section(provider) - if last_patt is None or last_patt != text: - view.select_first() + # if last_patt is None or last_patt != text: + # view.select_first() last_patt = text bar.show() From 7dfc7f7fa20d1487fa6dabd901e8774b7cda8d15 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 27 May 2021 11:40:08 -0400 Subject: [PATCH 72/81] Factor chart selection into widget, cleanups, add resource links --- piker/ui/_search.py | 202 +++++++++++++++++++++----------------------- 1 file changed, 95 insertions(+), 107 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 9b698ef3..05320183 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -103,6 +103,18 @@ class CompleterView(QTreeView): # https://doc.qt.io/qt-5/model-view-programming.html # - MV tut: # https://doc.qt.io/qt-5/modelview.html + # - custome header view (for doing stuff like we have in kivy?): + # https://doc.qt.io/qt-5/qheaderview.html#moving-header-sections + + # TODO: selection model stuff for eventual aggregate feeds + # charting and mgmt; + # https://doc.qt.io/qt-5/qabstractitemview.html#setSelectionModel + # https://doc.qt.io/qt-5/qitemselectionmodel.html + # https://doc.qt.io/qt-5/modelview.html#3-2-working-with-selections + # https://doc.qt.io/qt-5/model-view-programming.html#handling-selections-of-items + + # TODO: mouse extended handling: + # https://doc.qt.io/qt-5/qabstractitemview.html#entered def __init__( self, @@ -126,6 +138,8 @@ class CompleterView(QTreeView): # self.setUniformRowHeights(True) # self.setColumnWidth(0, 3) + # self.setVerticalBarPolicy(Qt.ScrollBarAlwaysOff) + # self.setSizeAdjustPolicy(QAbstractScrollArea.AdjustIgnored) # ux settings self.setItemsExpandable(True) @@ -133,15 +147,6 @@ class CompleterView(QTreeView): self.setAnimated(False) self.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) - # TODO: this up front? - # self.setSelectionModel( - # QItemSelectionModel.ClearAndSelect | - # QItemSelectionModel.Rows - # ) - - # self.setVerticalBarPolicy(Qt.ScrollBarAlwaysOff) - # self.setSizeAdjustPolicy(QAbstractScrollArea.AdjustIgnored) - # column headers model.setHorizontalHeaderLabels(labels) @@ -150,47 +155,9 @@ class CompleterView(QTreeView): def on_pressed(self, idx: QModelIndex) -> None: search = self.parent() - value = search.get_current_item() - - if value is not None: - provider, symbol = value - chart = search.chart_app - - chart.load_symbol( - provider, - symbol, - 'info', - ) - - # fully qualified symbol name (SNS i guess is what we're - # making?) - fqsn = '.'.join([symbol, provider]).lower() - - # Re-order the symbol cache on the chart to display in - # LIFO order. this is normally only done internally by - # the chart on new symbols being loaded into memory - chart.set_chart_symbol(fqsn, chart.linkedcharts) - + search.chart_current_item(clear_to_cache=False) search.focus() - # def viewportSizeHint(self) -> QtCore.QSize: - # vps = super().viewportSizeHint() - # return QSize(vps.width(), _font.px_size * 6 * 2) - - # def sizeHint(self) -> QtCore.QSize: - # """Scale completion results up to 6/16 of window. - # """ - # # height = self.window().height() * 1/6 - # # psh.setHeight(self.dpi_font.px_size * 6) - # # print(_font.px_size) - # height = _font.px_size * 6 * 2 - # # the default here is just the vp size without scroll bar - # # https://doc.qt.io/qt-5/qabstractscrollarea.html#viewportSizeHint - # vps = self.viewportSizeHint() - # # print(f'h: {height}\n{vps}') - # # psh.setHeight(12) - # return QSize(-1, height) - def set_font_size(self, size: int = 18): # dpi_px_size = _font.px_size # print(size) @@ -373,22 +340,15 @@ class CompleterView(QTreeView): # print(f'removing {rows} from {section}') assert model.removeRows(0, rows, parent=idx) - # remove section as well + # remove section as well ? # model.removeRow(i, QModelIndex()) if status_field is not None: model.setItem(idx.row(), 1, QStandardItem(status_field)) + else: model.setItem(idx.row(), 1, QStandardItem()) - # XXX: not idea how to use this - # model.setItemData( - # idx, - # { - # 0: 'cache', - # 1: 'searching', - # } - # ) self.resize() return idx @@ -435,6 +395,11 @@ class CompleterView(QTreeView): self.expandAll() + # TODO: figure out if we can avoid this line in a better way + # such that "re-selection" doesn't happen tree-wise for each new + # sub-search: + # https://doc.qt.io/qt-5/model-view-programming.html#handling-selections-in-item-views + # XXX: THE BELOW LINE MUST BE CALLED. # this stuff is super finicky and if not done right will cause # Qt crashes out our buttz. it's required in order to get the @@ -462,6 +427,10 @@ class SearchBar(QtWidgets.QLineEdit): super().__init__(parent) + # self.setContextMenuPolicy(Qt.CustomContextMenu) + # self.customContextMenuRequested.connect(self.show_menu) + # self.setStyleSheet(f"font: 18px") + self.view: CompleterView = view self.dpi_font = font self.chart_app = parent_chart @@ -477,10 +446,6 @@ class SearchBar(QtWidgets.QLineEdit): # witty bit of margin self.setTextMargins(2, 2, 2, 2) - # self.setContextMenuPolicy(Qt.CustomContextMenu) - # self.customContextMenuRequested.connect(self.show_menu) - # self.setStyleSheet(f"font: 18px") - def focus(self) -> None: self.selectAll() self.show() @@ -508,12 +473,18 @@ class SearchBar(QtWidgets.QLineEdit): class SearchWidget(QtGui.QWidget): + '''Composed widget of ``SearchBar`` + ``CompleterView``. + + Includes helper methods for item management in the sub-widgets. + + ''' def __init__( self, chart_space: 'ChartSpace', # type: ignore # noqa columns: List[str] = ['src', 'symbol'], parent=None, - ): + + ) -> None: super().__init__(parent or chart_space) # size it as we specify @@ -529,10 +500,11 @@ class SearchWidget(QtGui.QWidget): self.vbox.setSpacing(4) # split layout for the (label:| search bar entry) - self.bar_hbox = QtGui.QHBoxLayout(self) + self.bar_hbox = QtGui.QHBoxLayout() self.bar_hbox.setContentsMargins(0, 0, 0, 0) self.bar_hbox.setSpacing(4) + # add label to left of search bar self.label = label = QtGui.QLabel(parent=self) label.setTextFormat(3) # markdown label.setFont(_font.font) @@ -546,9 +518,6 @@ class SearchWidget(QtGui.QWidget): self.bar_hbox.addWidget(label) - # https://doc.qt.io/qt-5/qlayout.html#SizeConstraint-enum - # self.vbox.setSizeConstraint(QLayout.SetMaximumSize) - self.view = CompleterView( parent=self, labels=columns, @@ -560,8 +529,6 @@ class SearchWidget(QtGui.QWidget): ) self.bar_hbox.addWidget(self.bar) - # self.vbox.addWidget(self.bar) - # self.vbox.setAlignment(self.bar, Qt.AlignTop | Qt.AlignRight) self.vbox.addLayout(self.bar_hbox) self.vbox.setAlignment(self.bar, Qt.AlignTop | Qt.AlignRight) @@ -595,6 +562,7 @@ class SearchWidget(QtGui.QWidget): # to figure out the desired field(s) # https://doc.qt.io/qt-5/qstandarditemmodel.html#itemFromIndex node = model.itemFromIndex(cidx.siblingAtColumn(1)) + if node: symbol = node.text() try: @@ -612,6 +580,52 @@ class SearchWidget(QtGui.QWidget): else: return None + def chart_current_item( + self, + clear_to_cache: bool = True, + ) -> Optional[str]: + '''Attempt to load and switch the current selected + completion result to the affiliated chart app. + + Return any loaded symbol + + ''' + value = self.get_current_item() + if value is None: + return None + + provider, symbol = value + chart = self.chart_app + + log.info(f'Requesting symbol: {symbol}.{provider}') + + chart.load_symbol( + provider, + symbol, + 'info', + ) + + # fully qualified symbol name (SNS i guess is what we're + # making?) + fqsn = '.'.join([symbol, provider]).lower() + + # Re-order the symbol cache on the chart to display in + # LIFO order. this is normally only done internally by + # the chart on new symbols being loaded into memory + chart.set_chart_symbol(fqsn, chart.linkedcharts) + + if clear_to_cache: + self.bar.clear() + self.view.set_section_entries( + 'cache', + values=list(reversed(chart._chart_cache)), + + # remove all other completion results except for cache + clear_all=True, + ) + + return fqsn + _search_active: trio.Event = trio.Event() _search_enabled: bool = False @@ -630,7 +644,9 @@ async def pack_matches( ) -> None: log.info(f'Searching {provider} for "{pattern}"') + if provider != 'cache': + # insert provider entries with search status view.set_section_entries( section=provider, values=[], @@ -645,13 +661,14 @@ async def pack_matches( # ensure ^ status is updated results = await search(pattern) - if provider != 'cache': + if provider != 'cache': # XXX: don't cache the cache results xD matches[(provider, pattern)] = results # print(f'results from {provider}: {results}') has_results[pattern].add(provider) if results: + # display completion results view.set_section_entries( section=provider, values=results, @@ -661,10 +678,9 @@ async def pack_matches( async def fill_results( search: SearchBar, - # multisearch: Callable[..., Awaitable], recv_chan: trio.abc.ReceiveChannel, - # kb debouncing pauses + # kb debouncing pauses (bracket defaults) min_pause_time: float = 0.0616, max_pause_time: float = 6/16, @@ -736,7 +752,9 @@ async def fill_results( # "searching.." statuses on outstanding results providers async with trio.open_nursery() as n: - for provider, (search, pause) in _searcher_cache.copy().items(): + for provider, (search, pause) in ( + _searcher_cache.copy().items() + ): if provider != 'cache': view.clear_section( @@ -766,8 +784,8 @@ async def fill_results( else: view.clear_section(provider) - # if last_patt is None or last_patt != text: - # view.select_first() + if last_patt is None or last_patt != text: + view.select_first() last_patt = text bar.show() @@ -777,7 +795,6 @@ async def handle_keyboard_input( search: SearchWidget, recv_chan: trio.abc.ReceiveChannel, - keyboard_pause_period: float = 0.0616, ) -> None: @@ -819,36 +836,7 @@ async def handle_keyboard_input( if key in (Qt.Key_Enter, Qt.Key_Return): - value = search.get_current_item() - if value is None: - continue - - provider, symbol = value - - log.info(f'Requesting symbol: {symbol}.{provider}') - - chart.load_symbol( - provider, - symbol, - 'info', - ) - - # fully qualified symbol name (SNS i guess is what we're - # making?) - fqsn = '.'.join([symbol, provider]).lower() - - # Re-order the symbol cache on the chart to display in - # LIFO order. this is normally only done internally by - # the chart on new symbols being loaded into memory - chart.set_chart_symbol(fqsn, chart.linkedcharts) - - search.bar.clear() - view.set_section_entries( - 'cache', - values=list(reversed(chart._chart_cache)), - clear_all=True, - ) - + search.chart_current_item(clear_to_cache=True) _search_enabled = False continue @@ -893,12 +881,12 @@ async def handle_keyboard_input( view.next_section(direction='up') # selection navigation controls - elif ctl and key in { + elif (ctl and key in { Qt.Key_K, Qt.Key_J, - } or key in { + }) or key in { Qt.Key_Up, Qt.Key_Down, From 3e39e9620c20a8b292c619e6abbcef58419787b4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 27 May 2021 11:40:51 -0400 Subject: [PATCH 73/81] Add a no data available error --- piker/brokers/_util.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/piker/brokers/_util.py b/piker/brokers/_util.py index 64f0ad3a..4b4bd1cd 100644 --- a/piker/brokers/_util.py +++ b/piker/brokers/_util.py @@ -32,6 +32,10 @@ class SymbolNotFound(BrokerError): "Symbol not found by broker search" +class NoData(BrokerError): + "Symbol data not permitted" + + def resproc( resp: asks.response_objects.Response, log: logging.Logger, From e88e5b8ce2a4eaabfb06ea2bf12856fe6f395c76 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 27 May 2021 11:41:48 -0400 Subject: [PATCH 74/81] Decrease binance search debounce period --- piker/brokers/binance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/brokers/binance.py b/piker/brokers/binance.py index 1820b6c6..6f3014d1 100644 --- a/piker/brokers/binance.py +++ b/piker/brokers/binance.py @@ -82,7 +82,7 @@ _ohlc_dtype = [ ohlc_dtype = np.dtype(_ohlc_dtype) _show_wap_in_history = False -_search_conf = {'pause_period': 0.375} +_search_conf = {'pause_period': 0.0616} # https://binance-docs.github.io/apidocs/spot/en/#exchange-information @@ -298,7 +298,7 @@ async def stream_messages(ws): if cs.cancelled_caught: timeouts += 1 - if timeouts > 2: + if timeouts > 10: raise trio.TooSlowError("binance feed seems down?") continue From ec6ea32ddaa1fb400a90ea232d8c59c9285d9bc4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 27 May 2021 11:42:35 -0400 Subject: [PATCH 75/81] Don't pass through linked charts x-axis handle --- piker/ui/_chart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_chart.py b/piker/ui/_chart.py index d3661922..b92e4aa5 100644 --- a/piker/ui/_chart.py +++ b/piker/ui/_chart.py @@ -309,7 +309,7 @@ class LinkedSplitCharts(QtGui.QWidget): self.chart = self.add_plot( name=symbol.key, array=array, - xaxis=self.xaxis, + # xaxis=self.xaxis, style=style, _is_main=True, ) From 50aff72f8e7dd66512c86186a51a247bd9c25b92 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 27 May 2021 11:43:07 -0400 Subject: [PATCH 76/81] Don't require map (yet) in backend modules --- piker/data/feed.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/piker/data/feed.py b/piker/data/feed.py index 1097de50..a0e2478b 100644 --- a/piker/data/feed.py +++ b/piker/data/feed.py @@ -419,8 +419,12 @@ async def install_brokerd_search( provider_name=brokermod.name, search_routine=search, - pause_period=brokermod._search_conf.get('pause_period', 0.0616), + # TODO: should be make this a required attr of + # a backend module? + pause_period=getattr( + brokermod, '_search_conf', {} + ).get('pause_period', 0.0616), ): yield From c56c7b85402ae60d8f7613bb15d01ef0916c02e4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 28 May 2021 09:41:24 -0400 Subject: [PATCH 77/81] Increase min debounce period, stop searching on user nav selection --- piker/ui/_search.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 05320183..96e76478 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -404,8 +404,17 @@ class CompleterView(QTreeView): # this stuff is super finicky and if not done right will cause # Qt crashes out our buttz. it's required in order to get the # view to show right after typing input. + self.select_first() + # TODO: the way we might be able to do this more sanely is, + # 1. for the currently selected item, when start rewriting + # a section figure out the row, column, parent "abstract" + # position in the tree view and store it + # 2. take that position and re-apply the selection to the new + # model/tree by looking up the new "equivalent element" and + # selecting + self.show_matches() def show_matches(self) -> None: @@ -681,7 +690,7 @@ async def fill_results( recv_chan: trio.abc.ReceiveChannel, # kb debouncing pauses (bracket defaults) - min_pause_time: float = 0.0616, + min_pause_time: float = 0.1, max_pause_time: float = 6/16, ) -> None: @@ -697,7 +706,6 @@ async def fill_results( last_text = bar.text() repeats = 0 - last_patt = None # cache of prior patterns to search results matches = defaultdict(list) @@ -776,7 +784,7 @@ async def fill_results( ) else: # already has results for this input text results = matches[(provider, text)] - if results: + if results and provider != 'cache': view.set_section_entries( section=provider, values=results, @@ -784,10 +792,6 @@ async def fill_results( else: view.clear_section(provider) - if last_patt is None or last_patt != text: - view.select_first() - - last_patt = text bar.show() @@ -874,11 +878,13 @@ async def handle_keyboard_input( Qt.Key_D, }: view.next_section(direction='down') + _search_enabled = False elif ctl and key in { Qt.Key_U, }: view.next_section(direction='up') + _search_enabled = False # selection navigation controls elif (ctl and key in { @@ -903,14 +909,8 @@ async def handle_keyboard_input( if parent_item and parent_item.text() == 'cache': - value = search.get_current_item() - if value is not None: - provider, symbol = value - chart.load_symbol( - provider, - symbol, - 'info', - ) + # if it's a cache item, switch and show it immediately + search.chart_current_item(clear_to_cache=False) elif not ctl: # relay to completer task @@ -950,7 +950,7 @@ async def register_symbol_search( global _searcher_cache - pause_period = pause_period or 0.061 + pause_period = pause_period or 0.125 # deliver search func to consumer try: From 7fa9f3f542c273483a156682a022268bc1d9f477 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 28 May 2021 12:29:58 -0400 Subject: [PATCH 78/81] Add `Client.search_symbols()` to all backends, use it in `piker search` --- piker/_daemon.py | 8 +++++++- piker/brokers/binance.py | 19 ++++++++++++++++++ piker/brokers/cli.py | 24 +++++++++++++++++------ piker/brokers/core.py | 40 ++++++++++++++++++++++++++++++++++---- piker/brokers/ib.py | 24 ++++++++++++++++++++++- piker/brokers/kraken.py | 2 +- piker/brokers/questrade.py | 2 +- 7 files changed, 105 insertions(+), 14 deletions(-) diff --git a/piker/_daemon.py b/piker/_daemon.py index 4c37812d..07a584c3 100644 --- a/piker/_daemon.py +++ b/piker/_daemon.py @@ -142,8 +142,14 @@ async def maybe_open_runtime( Start the ``tractor`` runtime (a root actor) if none exists. """ + settings = _tractor_kwargs + settings.update(kwargs) + if not tractor.current_actor(err_on_no_runtime=False): - async with tractor.open_root_actor(loglevel=loglevel, **kwargs): + async with tractor.open_root_actor( + loglevel=loglevel, + **settings, + ): yield else: yield diff --git a/piker/brokers/binance.py b/piker/brokers/binance.py index 6f3014d1..c2de7ded 100644 --- a/piker/brokers/binance.py +++ b/piker/brokers/binance.py @@ -207,6 +207,25 @@ class Client: return self._pairs + async def search_symbols( + self, + pattern: str, + limit: int = None, + ) -> Dict[str, Any]: + if self._pairs is not None: + data = self._pairs + else: + data = await self.symbol_info() + + matches = fuzzy.extractBests( + pattern, + data, + score_cutoff=50, + ) + # repack in dict form + return {item[0]['symbol']: item[0] + for item in matches} + async def bars( self, symbol: str, diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py index b0083cfa..164a060b 100644 --- a/piker/brokers/cli.py +++ b/piker/brokers/cli.py @@ -30,7 +30,7 @@ import tractor from ..cli import cli from .. import watchlists as wl from ..log import get_console_log, colorize_json, get_logger -from .._daemon import maybe_spawn_brokerd +from .._daemon import maybe_spawn_brokerd, maybe_open_pikerd from ..brokers import core, get_brokermod, data log = get_logger('cli') @@ -273,13 +273,25 @@ def search(config, pattern): """Search for symbols from broker backend(s). """ # global opts - brokermod = config['brokermods'][0] + brokermods = config['brokermods'] - quotes = tractor.run( - partial(core.symbol_search, brokermod, pattern), - start_method='forkserver', - loglevel='info', + # define tractor entrypoint + async def main(func): + + async with maybe_open_pikerd( + loglevel=config['loglevel'], + ): + return await func() + + quotes = trio.run( + main, + partial( + core.symbol_search, + brokermods, + pattern, + ), ) + if not quotes: log.error(f"No matches could be found for {pattern}?") return diff --git a/piker/brokers/core.py b/piker/brokers/core.py index 5189df85..59621b63 100644 --- a/piker/brokers/core.py +++ b/piker/brokers/core.py @@ -24,8 +24,12 @@ import inspect from types import ModuleType from typing import List, Dict, Any, Optional +import trio + from ..log import get_logger from . import get_brokermod +from .._daemon import maybe_spawn_brokerd +from .api import open_cached_client log = get_logger(__name__) @@ -126,13 +130,41 @@ async def symbol_info( return await client.symbol_info(symbol, **kwargs) +async def search_w_brokerd(name: str, pattern: str) -> dict: + + async with open_cached_client(name) as client: + + # TODO: support multiple asset type concurrent searches. + return await client.search_symbols(pattern=pattern) + + async def symbol_search( - brokermod: ModuleType, + brokermods: list[ModuleType], pattern: str, **kwargs, ) -> Dict[str, Dict[str, Dict[str, Any]]]: """Return symbol info from broker. """ - async with brokermod.get_client() as client: - # TODO: support multiple asset type concurrent searches. - return await client.search_stocks(pattern=pattern, **kwargs) + results = [] + + async def search_backend(brokername: str) -> None: + + async with maybe_spawn_brokerd( + brokername, + ) as portal: + + results.append(( + brokername, + await portal.run( + search_w_brokerd, + name=brokername, + pattern=pattern, + ), + )) + + async with trio.open_nursery() as n: + + for mod in brokermods: + n.start_soon(search_backend, mod.name) + + return results diff --git a/piker/brokers/ib.py b/piker/brokers/ib.py index 189f800d..96398c44 100644 --- a/piker/brokers/ib.py +++ b/piker/brokers/ib.py @@ -52,7 +52,7 @@ from ..log import get_logger, get_console_log from .._daemon import maybe_spawn_brokerd from ..data._source import from_df from ..data._sharedmem import ShmArray -from ._util import SymbolNotFound +from ._util import SymbolNotFound, NoData log = get_logger(__name__) @@ -311,6 +311,18 @@ class Client: else: return {} + async def search_symbols( + self, + pattern: str, + # how many contracts to search "up to" + upto: int = 3, + asdicts: bool = True, + ) -> Dict[str, ContractDetails]: + + # TODO add search though our adhoc-locally defined symbol set + # for futes/cmdtys/ + return await self.search_stocks(pattern, upto, asdicts) + async def search_futes( self, pattern: str, @@ -862,6 +874,13 @@ async def get_bars( # throttling despite the rps being low break + elif 'No market data permissions for' in err.message: + + # TODO: signalling for no permissions searches + raise NoData(f'Symbol: {sym}') + break + + else: log.exception( "Data query rate reached: Press `ctrl-alt-f`" @@ -1133,8 +1152,10 @@ async def stream_quotes( # tell caller quotes are now coming in live feed_is_live.set() + # last = time.time() async with aclosing(stream): async for ticker in stream: + # print(f'ticker rate: {1/(time.time() - last)}') # print(ticker.vwap) quote = normalize( @@ -1149,6 +1170,7 @@ async def stream_quotes( # ugh, clear ticks since we've consumed them ticker.ticks = [] + # last = time.time() def pack_position(pos: Position) -> Dict[str, Any]: diff --git a/piker/brokers/kraken.py b/piker/brokers/kraken.py index f2adccf4..7e9afd5c 100644 --- a/piker/brokers/kraken.py +++ b/piker/brokers/kraken.py @@ -207,7 +207,7 @@ class Client: return self._pairs - async def search_stocks( + async def search_symbols( self, pattern: str, limit: int = None, diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 528df91e..3f09587c 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -628,7 +628,7 @@ class Client: f"Took {time.time() - start} seconds to retreive {len(bars)} bars") return bars - async def search_stocks( + async def search_symbols( self, pattern: str, # how many contracts to return From ea3d96e7ed8a629130816ddc153b46830cc7de35 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 28 May 2021 12:55:11 -0400 Subject: [PATCH 79/81] Accept arbitrary QEvent subscriptions via a set --- piker/ui/_event.py | 26 +++++++++++++++----------- piker/ui/_search.py | 2 +- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/piker/ui/_event.py b/piker/ui/_event.py index ac66417f..c3d919dc 100644 --- a/piker/ui/_event.py +++ b/piker/ui/_event.py @@ -30,7 +30,7 @@ class EventCloner(QtCore.QObject): for later async processing. """ - + _event_types: set[QEvent] = set() _send_chan: trio.abc.SendChannel = None def eventFilter( @@ -39,14 +39,11 @@ class EventCloner(QtCore.QObject): ev: QEvent, ) -> None: - if ev.type() in { - QEvent.KeyPress, - # QEvent.KeyRelease, - }: - # TODO: is there a global setting for this? - if ev.isAutoRepeat(): - ev.ignore() - return False + if ev.type() in self._event_types: + + # TODO: what's the right way to allow this? + # if ev.isAutoRepeat(): + # ev.ignore() # XXX: we unpack here because apparently doing it # after pop from the mem chan isn't showing the same @@ -59,7 +56,7 @@ class EventCloner(QtCore.QObject): txt = ev.text() # run async processing - self._send_chan.send_nowait((key, mods, txt)) + self._send_chan.send_nowait((ev, key, mods, txt)) # never intercept the event return False @@ -69,7 +66,11 @@ class EventCloner(QtCore.QObject): async def open_key_stream( source_widget: QtGui.QWidget, - event_type: QEvent = QEvent.KeyPress, + event_types: set[QEvent] = {QEvent.KeyPress}, + + # TODO: should we offer some kinda option for toggling releases? + # would it require a channel per event type? + # QEvent.KeyRelease, ) -> trio.abc.ReceiveChannel: @@ -78,10 +79,13 @@ async def open_key_stream( kc = EventCloner() kc._send_chan = send + kc._event_types = event_types + source_widget.installEventFilter(kc) try: yield recv + finally: await send.aclose() source_widget.removeEventFilter(kc) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 96e76478..61ee4a75 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -825,7 +825,7 @@ async def handle_keyboard_input( ) ) - async for key, mods, txt in recv_chan: + async for event, key, mods, txt in recv_chan: log.debug(f'key: {key}, mods: {mods}, txt: {txt}') From a31b83c5ca936686d06ed4ed9a75bdea639281e9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 28 May 2021 13:44:30 -0400 Subject: [PATCH 80/81] Don't ever send plain whitespace a search pattern --- piker/ui/_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 61ee4a75..8d82ddfd 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -735,7 +735,7 @@ async def fill_results( text = bar.text() # print(f'search: {text}') - if not text: + if not text or text.isspace(): # print('idling') _search_active = trio.Event() break From ee71f445fba343195290106e16a28ce1a3e4ee11 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 28 May 2021 14:08:24 -0400 Subject: [PATCH 81/81] Clear entries on no results returned per task --- piker/ui/_search.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/piker/ui/_search.py b/piker/ui/_search.py index 8d82ddfd..3adb16a1 100644 --- a/piker/ui/_search.py +++ b/piker/ui/_search.py @@ -682,6 +682,8 @@ async def pack_matches( section=provider, values=results, ) + else: + view.clear_section(provider) async def fill_results(