From e45cdf92f055135f30fda43bd4adbd69dc78da4f Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 23 Jan 2018 01:03:51 -0500 Subject: [PATCH 01/12] Log entire access config on exit --- piker/brokers/__init__.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/piker/brokers/__init__.py b/piker/brokers/__init__.py index f2c6409c..a78fb558 100644 --- a/piker/brokers/__init__.py +++ b/piker/brokers/__init__.py @@ -1,14 +1,15 @@ """ -Broker client-daemons and general back end machinery. +Broker clients, daemons and general back end machinery. """ import sys import trio +from pprint import pformat from .questrade import serve_forever from ..log import get_console_log def main() -> None: - log = get_console_log('INFO', name='questrade') + log = get_console_log('info', name='questrade') argv = sys.argv[1:] refresh_token = None @@ -21,7 +22,4 @@ def main() -> None: except Exception as err: log.exception(err) else: - log.info( - f"\nLast refresh_token: {client.access_data['refresh_token']}\n" - f"Last access_token: {client.access_data['access_token']}\n" - ) + log.debug(f"Exiting with last access info:\n{pformat(client.access_data)}\n") From 1b0269e51aec06e02174c07c25d7fc2b1c4a0e45 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Tue, 23 Jan 2018 01:05:02 -0500 Subject: [PATCH 02/12] Drop `Client.from_config()` factory - more cleanups --- piker/brokers/questrade.py | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index bdb5e710..3e9839b8 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -25,7 +25,9 @@ def resproc( resp: asks.response_objects.Response, return_json: bool = True ) -> asks.response_objects.Response: - """Raise error on non-200 OK response. + """Process response and return its json content. + + Raise the appropriate error on non-200 OK responses. """ data = resp.json() log.debug(f"Received json contents:\n{pformat(data)}\n") @@ -57,18 +59,10 @@ class Client: """API client suitable for use as a long running broker daemon. """ def __init__(self, config: dict): - sess = self._sess = asks.Session() - self.api = API(sess) + self._sess = asks.Session() + self.api = API(self._sess) self.access_data = config self.user_data = {} - self._conf = None # possibly set in ``from_config`` factory - - @classmethod - async def from_config(cls, config): - client = cls(dict(config['questrade'])) - client._conf = config - await client.enable_access() - return client async def _new_auth_token(self) -> dict: """Request a new api authorization ``refresh_token``. @@ -89,7 +83,7 @@ class Client: return data - async def _prep_sess(self) -> None: + def _prep_sess(self) -> None: """Fill http session with auth headers and a base url. """ data = self.access_data @@ -119,19 +113,16 @@ class Client: """ access_token = self.access_data.get('access_token') expires = float(self.access_data.get('expires_at', 0)) - # expired_by = time.time() - float(self.ttl or 0) - # if not access_token or (self.ttl is None) or (expires < time.time()): if not access_token or (expires < time.time()) or force_refresh: - log.info( - f"Access token {access_token} expired @ {expires}, " - "refreshing...") + log.info(f"Refreshing access token {access_token} which expired at" + f" {expires}") data = await self._new_auth_token() # store absolute token expiry time self.access_data['expires_at'] = time.time() + float( data['expires_in']) - await self._prep_sess() + self._prep_sess() return self.access_data @@ -141,7 +132,7 @@ def get_config() -> "configparser.ConfigParser": not conf['questrade'].get('refresh_token') ): log.warn( - f"No valid `questrade` refresh token could be found in {path}") + f"No valid refresh token could be found in {path}") # get from user refresh_token = input("Please provide your Questrade access token: ") conf['questrade'] = {'refresh_token': refresh_token} @@ -152,12 +143,11 @@ def get_config() -> "configparser.ConfigParser": @asynccontextmanager async def get_client(refresh_token: str = None) -> Client: """Spawn a broker client. - """ conf = get_config() - log.debug(f"Loaded questrade config: {conf['questrade']}") - log.info("Waiting on api access...") - client = await Client.from_config(conf) + log.debug(f"Loaded config:\n{pformat(dict(conf['questrade']))}\n") + client = Client(dict(conf['questrade'])) + await client.enable_access() try: try: # do a test ping to ensure the access token works @@ -170,6 +160,8 @@ async def get_client(refresh_token: str = None) -> Client: await client.enable_access(force_refresh=True) await client.api.time() + accounts = await client.api.accounts() + log.info(f"Available accounts:\n{pformat(accounts)}\n") yield client finally: # save access creds for next run From 5c4996873a5042ec0894205fd55e86586ddcc2d6 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 25 Jan 2018 20:54:13 -0500 Subject: [PATCH 03/12] Start using click for cli --- piker/brokers/__init__.py | 22 ---------------------- piker/brokers/cli.py | 29 +++++++++++++++++++++++++++++ piker/brokers/questrade.py | 18 ++++++++++++------ setup.py | 6 ++++-- 4 files changed, 45 insertions(+), 30 deletions(-) create mode 100644 piker/brokers/cli.py diff --git a/piker/brokers/__init__.py b/piker/brokers/__init__.py index a78fb558..54afc783 100644 --- a/piker/brokers/__init__.py +++ b/piker/brokers/__init__.py @@ -1,25 +1,3 @@ """ Broker clients, daemons and general back end machinery. """ -import sys -import trio -from pprint import pformat -from .questrade import serve_forever -from ..log import get_console_log - - -def main() -> None: - log = get_console_log('info', name='questrade') - argv = sys.argv[1:] - - refresh_token = None - if argv: - refresh_token = argv[0] - - # main loop - try: - client = trio.run(serve_forever, refresh_token) - except Exception as err: - log.exception(err) - else: - log.debug(f"Exiting with last access info:\n{pformat(client.access_data)}\n") diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py new file mode 100644 index 00000000..2980d785 --- /dev/null +++ b/piker/brokers/cli.py @@ -0,0 +1,29 @@ +""" +Console interface to broker client/daemons. +""" +from pprint import pformat +import click +import trio +from ..log import get_console_log + + +def run(loglevel, main): + log = get_console_log(loglevel) + + # main loop + try: + client = trio.run(main) + except Exception as err: + log.exception(err) + else: + log.debug( + f"Exiting with last access info:\n{pformat(client.access_data)}\n") + + +@click.command() +@click.option('--broker', default='questrade', help='Broker backend to use') +@click.option('--loglevel', '-l', default='warning', help='Logging level') +def pikerd(broker, loglevel): + # import broker module daemon entry point + from .questrade import serve_forever + run(loglevel, serve_forever) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 3e9839b8..6d65d44f 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -5,6 +5,7 @@ from . import config from ..log import get_logger from pprint import pformat import time +import datetime from async_generator import asynccontextmanager # TODO: move to urllib3/requests once supported @@ -113,14 +114,19 @@ class Client: """ access_token = self.access_data.get('access_token') expires = float(self.access_data.get('expires_at', 0)) + expires_stamp = datetime.datetime.fromtimestamp( + expires).strftime('%Y-%m-%d %H:%M:%S') if not access_token or (expires < time.time()) or force_refresh: log.info(f"Refreshing access token {access_token} which expired at" - f" {expires}") + f" {expires_stamp}") data = await self._new_auth_token() # store absolute token expiry time self.access_data['expires_at'] = time.time() + float( data['expires_in']) + else: + log.info(f"\nCurrent access token {access_token} expires at" + f" {expires_stamp}\n") self._prep_sess() return self.access_data @@ -141,7 +147,7 @@ def get_config() -> "configparser.ConfigParser": @asynccontextmanager -async def get_client(refresh_token: str = None) -> Client: +async def get_client() -> Client: """Spawn a broker client. """ conf = get_config() @@ -150,8 +156,8 @@ async def get_client(refresh_token: str = None) -> Client: await client.enable_access() try: - try: # do a test ping to ensure the access token works - log.debug("Check time to ensure access token is valid") + log.debug("Check time to ensure access token is valid") + try: await client.api.time() except Exception as err: # access token is likely no good @@ -169,10 +175,10 @@ async def get_client(refresh_token: str = None) -> Client: config.write(conf) -async def serve_forever(refresh_token: str = None) -> None: +async def serve_forever() -> None: """Start up a client and serve until terminated. """ - async with get_client(refresh_token) as client: + async with get_client() as client: # pretty sure this doesn't work # await client._revoke_auth_token() return client diff --git a/setup.py b/setup.py index ca2daa3e..ae606424 100755 --- a/setup.py +++ b/setup.py @@ -28,10 +28,12 @@ setup( ], entry_points={ 'console_scripts': [ - 'pikerd = piker.brokers:main', + 'pikerd = piker.brokers.cli:pikerd', ] }, - install_requires=['click', 'colorlog', 'trio', 'attrs'], + install_requires=[ + 'click', 'colorlog', 'trio', 'attrs', 'async_generator' + ], extras_require={ 'questrade': ['asks'], }, From c6cff5a4327b833bf09bffc093c86d7af2c664bb Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 25 Jan 2018 20:59:56 -0500 Subject: [PATCH 04/12] Swap debug-garbage log colours --- piker/log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/piker/log.py b/piker/log.py index 5433d69a..c4304c1f 100644 --- a/piker/log.py +++ b/piker/log.py @@ -32,9 +32,9 @@ STD_PALETTE = { 'ERROR': 'red', 'WARNING': 'yellow', 'INFO': 'green', - 'DEBUG': 'purple', + 'DEBUG': 'blue', 'TRACE': 'cyan', - 'GARBAGE': 'blue', + 'GARBAGE': 'purple', } BOLD_PALETTE = { 'bold': { From 4e1c64a7fba24a72e5d93a58fd85d2638b8159c0 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 25 Jan 2018 21:08:49 -0500 Subject: [PATCH 05/12] Import broker backend by name --- piker/brokers/cli.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py index 2980d785..e212831a 100644 --- a/piker/brokers/cli.py +++ b/piker/brokers/cli.py @@ -1,6 +1,7 @@ """ Console interface to broker client/daemons. """ +import importlib from pprint import pformat import click import trio @@ -22,8 +23,8 @@ def run(loglevel, main): @click.command() @click.option('--broker', default='questrade', help='Broker backend to use') -@click.option('--loglevel', '-l', default='warning', help='Logging level') +@click.option('--loglevel', '-l', default='info', help='Logging level') def pikerd(broker, loglevel): # import broker module daemon entry point - from .questrade import serve_forever - run(loglevel, serve_forever) + brokermod = importlib.import_module('.' + broker, 'piker.brokers') + run(loglevel, brokermod.serve_forever) From 9e8ed392d4ffca1be2096c671f687f5083809ce9 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Thu, 25 Jan 2018 21:53:55 -0500 Subject: [PATCH 06/12] Add token refresher task --- piker/brokers/questrade.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 6d65d44f..2595b818 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -1,6 +1,7 @@ """ Questrade API backend. """ +import trio from . import config from ..log import get_logger from pprint import pformat @@ -105,12 +106,14 @@ class Client: ) return resp - async def enable_access(self, force_refresh: bool = False) -> dict: - """Acquire new ``refresh_token`` and/or ``access_token`` if necessary. + async def ensure_access(self, force_refresh: bool = False) -> dict: + """Acquire new ``access_token`` and/or ``refresh_token`` if necessary. - Only needs to be called if the locally stored ``refresh_token`` has + Checks if the locally cached (file system) ``access_token`` has expired + (based on a ``expires_at`` time stamp stored in the brokers.ini config) expired (normally has a lifetime of 3 days). If ``false is set then - refresh the access token instead of using the locally cached version. + and refreshs token if necessary using the ``refresh_token``. If the + ``refresh_token`` has expired a new one needs to be provided by the user. """ access_token = self.access_data.get('access_token') expires = float(self.access_data.get('expires_at', 0)) @@ -146,6 +149,14 @@ def get_config() -> "configparser.ConfigParser": return conf +async def token_refresher(client): + """Coninually refresh the ``access_token`` near its expiry time. + """ + while True: + await trio.sleep(float(client.access_data['expires_at']) - time.time() - .1) + await client.ensure_access() + + @asynccontextmanager async def get_client() -> Client: """Spawn a broker client. @@ -153,7 +164,7 @@ async def get_client() -> Client: conf = get_config() log.debug(f"Loaded config:\n{pformat(dict(conf['questrade']))}\n") client = Client(dict(conf['questrade'])) - await client.enable_access() + await client.ensure_access() try: log.debug("Check time to ensure access token is valid") @@ -163,7 +174,7 @@ async def get_client() -> Client: # access token is likely no good log.warn(f"Access token {client.access_data['access_token']} seems" f" expired, forcing refresh") - await client.enable_access(force_refresh=True) + await client.ensure_access(force_refresh=True) await client.api.time() accounts = await client.api.accounts() @@ -181,4 +192,5 @@ async def serve_forever() -> None: async with get_client() as client: # pretty sure this doesn't work # await client._revoke_auth_token() - return client + async with trio.open_nursery() as nursery: + nursery.start_soon(token_refresher, client) From 534ba0b6980d69d2318bef7e25149a735082c0c4 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 26 Jan 2018 11:31:11 -0500 Subject: [PATCH 07/12] Add json highlighting; make debug msgs white --- piker/log.py | 20 +++++++++++++++++--- setup.py | 4 +++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/piker/log.py b/piker/log.py index c4304c1f..a849dbce 100644 --- a/piker/log.py +++ b/piker/log.py @@ -4,7 +4,9 @@ Log like a forester! """ import sys import logging +import json import colorlog +from pygments import highlight, lexers, formatters _proj_name = 'piker' @@ -12,7 +14,8 @@ _proj_name = 'piker' # (NOTE: we use the '{' format style) # Here, `thin_white` is just the laymen's gray. LOG_FORMAT = ( - "{bold_white}{thin_white}{asctime}{reset}" + # "{bold_white}{log_color}{asctime}{reset}" + "{log_color}{asctime}{reset}" " {bold_white}{thin_white}({reset}" "{thin_white}{threadName}{reset}{bold_white}{thin_white})" " {reset}{log_color}[{reset}{bold_log_color}{levelname}{reset}{log_color}]" @@ -32,9 +35,9 @@ STD_PALETTE = { 'ERROR': 'red', 'WARNING': 'yellow', 'INFO': 'green', - 'DEBUG': 'blue', + 'DEBUG': 'white', 'TRACE': 'cyan', - 'GARBAGE': 'purple', + 'GARBAGE': 'blue', } BOLD_PALETTE = { 'bold': { @@ -83,3 +86,14 @@ def get_console_log(level: str = None, name: str = None) -> logging.Logger: log.addHandler(handler) return log + + +def colorize_json(data, style='algol_nu'): + """Colorize json output using ``pygments``. + """ + formatted_json = json.dumps(data, sort_keys=True, indent=4) + return highlight( + formatted_json, lexers.JsonLexer(), + # likeable styles: algol_nu, tango, monokai + formatters.TerminalTrueColorFormatter(style=style) + ) diff --git a/setup.py b/setup.py index ae606424..f5edd117 100755 --- a/setup.py +++ b/setup.py @@ -29,10 +29,12 @@ setup( entry_points={ 'console_scripts': [ 'pikerd = piker.brokers.cli:pikerd', + 'piker = piker.brokers.cli:cli', ] }, install_requires=[ - 'click', 'colorlog', 'trio', 'attrs', 'async_generator' + 'click', 'colorlog', 'trio', 'attrs', 'async_generator', + 'pygments', ], extras_require={ 'questrade': ['asks'], From 27a39ac3ada57565c2909cef57d20bbcd50b5e28 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 26 Jan 2018 14:25:53 -0500 Subject: [PATCH 08/12] More client improvements - colorize json response data in logs - support ``refresh_token`` retrieval from user if the token for some reason expires while the client is live - extend api method support for markets, search, symbols, and quotes - support "proxying" through api calls via an ``api`` coro for one off client queries (useful for cli testing) --- piker/brokers/questrade.py | 143 ++++++++++++++++++++++++++----------- 1 file changed, 101 insertions(+), 42 deletions(-) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index 2595b818..b7ad9d7c 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -1,14 +1,16 @@ """ Questrade API backend. """ -import trio -from . import config -from ..log import get_logger -from pprint import pformat +import json import time import datetime + +import trio from async_generator import asynccontextmanager +from . import config +from ..log import get_logger, colorize_json + # TODO: move to urllib3/requests once supported import asks asks.init('trio') @@ -31,40 +33,32 @@ def resproc( Raise the appropriate error on non-200 OK responses. """ - data = resp.json() - log.debug(f"Received json contents:\n{pformat(data)}\n") - if not resp.status_code == 200: raise QuestradeError(resp.body) + try: + data = resp.json() + except json.decoder.JSONDecodeError: + log.exception(f"Failed to process {resp}") + else: + log.debug(f"Received json contents:\n{colorize_json(data)}") + return data if return_json else resp -class API: - """Questrade API at its finest. - """ - def __init__(self, session: asks.Session): - self._sess = session - - async def _request(self, path: str) -> dict: - resp = await self._sess.get(path=f'/{path}') - return resproc(resp) - - async def accounts(self): - return await self._request('accounts') - - async def time(self): - return await self._request('time') - - class Client: """API client suitable for use as a long running broker daemon. """ - def __init__(self, config: dict): + def __init__(self, config: 'configparser.ConfigParser'): self._sess = asks.Session() self.api = API(self._sess) - self.access_data = config + self._conf = config + self.access_data = {} self.user_data = {} + self._apply_config(config) + + def _apply_config(self, config): + self.access_data = dict(self._conf['questrade']) async def _new_auth_token(self) -> dict: """Request a new api authorization ``refresh_token``. @@ -113,7 +107,8 @@ class Client: (based on a ``expires_at`` time stamp stored in the brokers.ini config) expired (normally has a lifetime of 3 days). If ``false is set then and refreshs token if necessary using the ``refresh_token``. If the - ``refresh_token`` has expired a new one needs to be provided by the user. + ``refresh_token`` has expired a new one needs to be provided by the + user. """ access_token = self.access_data.get('access_token') expires = float(self.access_data.get('expires_at', 0)) @@ -122,11 +117,20 @@ class Client: if not access_token or (expires < time.time()) or force_refresh: log.info(f"Refreshing access token {access_token} which expired at" f" {expires_stamp}") - data = await self._new_auth_token() + try: + data = await self._new_auth_token() + except QuestradeError as qterr: + # likely config ``refresh_token`` is expired + if qterr.args[0].decode() == 'Bad Request': + _token_from_user(self._conf) + self._apply_config(self._conf) + data = await self._new_auth_token() # store absolute token expiry time self.access_data['expires_at'] = time.time() + float( data['expires_in']) + # write to config on disk + write_conf(self) else: log.info(f"\nCurrent access token {access_token} expires at" f" {expires_stamp}\n") @@ -135,6 +139,53 @@ class Client: return self.access_data +class API: + """Questrade API at its finest. + """ + def __init__(self, session: asks.Session): + self._sess = session + + async def _request(self, path: str, params=None) -> dict: + resp = await self._sess.get(path=f'/{path}', params=params) + return resproc(resp) + + async def accounts(self) -> dict: + return await self._request('accounts') + + async def time(self) -> dict: + return await self._request('time') + + async def markets(self) -> dict: + return await self._request('markets') + + async def search(self, prefix: str) -> dict: + return await self._request( + 'symbols/search', params={'prefix': prefix}) + + async def symbols(self, ids: str = '', names: str = '') -> dict: + log.debug(f"Symbol lookup for {ids}") + return await self._request( + 'symbols', params={'ids': ids, 'names': names}) + + async def quotes(self, ids: str) -> dict: + return await self._request('markets/quotes', params={'ids': ids}) + + +async def token_refresher(client): + """Coninually refresh the ``access_token`` near its expiry time. + """ + while True: + await trio.sleep( + float(client.access_data['expires_at']) - time.time() - .1) + await client.ensure_access(force_refresh=True) + + +def _token_from_user(conf: 'configparser.ConfigParser') -> None: + # get from user + refresh_token = input("Please provide your Questrade access token: ") + conf['questrade'] = {'refresh_token': refresh_token} + + def get_config() -> "configparser.ConfigParser": conf, path = config.load() if not conf.has_section('questrade') or ( @@ -142,19 +193,16 @@ def get_config() -> "configparser.ConfigParser": ): log.warn( f"No valid refresh token could be found in {path}") - # get from user - refresh_token = input("Please provide your Questrade access token: ") - conf['questrade'] = {'refresh_token': refresh_token} + _token_from_user(conf) return conf -async def token_refresher(client): - """Coninually refresh the ``access_token`` near its expiry time. +def write_conf(client): + """Save access creds to config file. """ - while True: - await trio.sleep(float(client.access_data['expires_at']) - time.time() - .1) - await client.ensure_access() + client._conf['questrade'] = client.access_data + config.write(client._conf) @asynccontextmanager @@ -162,8 +210,8 @@ async def get_client() -> Client: """Spawn a broker client. """ conf = get_config() - log.debug(f"Loaded config:\n{pformat(dict(conf['questrade']))}\n") - client = Client(dict(conf['questrade'])) + log.debug(f"Loaded config:\n{colorize_json(dict(conf['questrade']))}") + client = Client(conf) await client.ensure_access() try: @@ -178,12 +226,10 @@ async def get_client() -> Client: await client.api.time() accounts = await client.api.accounts() - log.info(f"Available accounts:\n{pformat(accounts)}\n") + log.info(f"Available accounts:\n{colorize_json(accounts)}") yield client finally: - # save access creds for next run - conf['questrade'] = client.access_data - config.write(conf) + write_conf(client) async def serve_forever() -> None: @@ -194,3 +240,16 @@ async def serve_forever() -> None: # await client._revoke_auth_token() async with trio.open_nursery() as nursery: nursery.start_soon(token_refresher, client) + + +async def api(methname, **kwargs) -> dict: + async with get_client() as client: + meth = getattr(client.api, methname, None) + if meth is None: + log.error(f"No api method `{methname}` could be found?") + else: + arg = kwargs.get('arg') + if arg: + return await meth(arg) + else: + return await meth(**kwargs) From 1b93a4c02aa1e8220161268867a69b338b57ab6c Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Fri, 26 Jan 2018 14:31:15 -0500 Subject: [PATCH 09/12] Add an `api` cli subcommand for console testing Add `piker api ` for easy testing of the underlying broker api from the console. --- piker/brokers/cli.py | 53 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py index e212831a..492edb96 100644 --- a/piker/brokers/cli.py +++ b/piker/brokers/cli.py @@ -1,24 +1,26 @@ """ Console interface to broker client/daemons. """ -import importlib -from pprint import pformat +import json +from functools import partial +from importlib import import_module + import click import trio -from ..log import get_console_log + +from ..log import get_console_log, colorize_json -def run(loglevel, main): +def run(main, loglevel='info'): log = get_console_log(loglevel) - # main loop + # main sandwich try: - client = trio.run(main) + return trio.run(main) except Exception as err: log.exception(err) - else: - log.debug( - f"Exiting with last access info:\n{pformat(client.access_data)}\n") + finally: + log.debug("Exiting pikerd") @click.command() @@ -26,5 +28,34 @@ def run(loglevel, main): @click.option('--loglevel', '-l', default='info', help='Logging level') def pikerd(broker, loglevel): # import broker module daemon entry point - brokermod = importlib.import_module('.' + broker, 'piker.brokers') - run(loglevel, brokermod.serve_forever) + brokermod = import_module('.' + broker, 'piker.brokers') + run(brokermod.serve_forever, loglevel) + + +@click.group() +def cli(): + pass + + +@cli.command() +@click.option('--broker', default='questrade', help='Broker backend to use') +@click.option('--loglevel', '-l', default='warning', help='Logging level') +@click.argument('meth', nargs=1) +@click.argument('kwargs', nargs=-1, required=True) +def api(meth, kwargs, loglevel, broker): + """Client for testing broker API methods with pretty printing of output. + """ + log = get_console_log(loglevel) + brokermod = import_module('.' + broker, 'piker.brokers') + + _kwargs = {} + for kwarg in kwargs: + if '=' not in kwarg: + log.error(f"kwarg `{kwarg}` must be of form =") + else: + key, _, value = kwarg.partition('=') + _kwargs[key] = value + + data = run(partial(brokermod.api, meth, **_kwargs), loglevel=loglevel) + if data: + click.echo(colorize_json(data)) From 66441d15e8bb5f1df6da7ebba46a1e3074630a56 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 27 Jan 2018 01:52:00 -0500 Subject: [PATCH 10/12] Complain when kwargs are missing but required --- piker/brokers/cli.py | 6 +++--- piker/brokers/config.py | 1 - piker/brokers/questrade.py | 23 ++++++++++++++++------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py index 492edb96..11678f7f 100644 --- a/piker/brokers/cli.py +++ b/piker/brokers/cli.py @@ -20,7 +20,7 @@ def run(main, loglevel='info'): except Exception as err: log.exception(err) finally: - log.debug("Exiting pikerd") + log.debug("Exiting piker") @click.command() @@ -41,9 +41,9 @@ def cli(): @click.option('--broker', default='questrade', help='Broker backend to use') @click.option('--loglevel', '-l', default='warning', help='Logging level') @click.argument('meth', nargs=1) -@click.argument('kwargs', nargs=-1, required=True) +@click.argument('kwargs', nargs=-1) def api(meth, kwargs, loglevel, broker): - """Client for testing broker API methods with pretty printing of output. + """client for testing broker API methods with pretty printing of output. """ log = get_console_log(loglevel) brokermod = import_module('.' + broker, 'piker.brokers') diff --git a/piker/brokers/config.py b/piker/brokers/config.py index 5d923adf..979e2605 100644 --- a/piker/brokers/config.py +++ b/piker/brokers/config.py @@ -16,7 +16,6 @@ def load() -> (configparser.ConfigParser, str): Create a ``broker.ini`` file if one dne. """ config = configparser.ConfigParser() - # mode = 'r' if path.exists(_broker_conf_path) else 'a' read = config.read(_broker_conf_path) log.debug(f"Read config file {_broker_conf_path}") return config, _broker_conf_path diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index b7ad9d7c..f2661a73 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -1,6 +1,7 @@ """ Questrade API backend. """ +import inspect import json import time import datetime @@ -47,7 +48,8 @@ def resproc( class Client: - """API client suitable for use as a long running broker daemon. + """API client suitable for use as a long running broker daemon or + for single api requests. """ def __init__(self, config: 'configparser.ConfigParser'): self._sess = asks.Session() @@ -243,13 +245,20 @@ async def serve_forever() -> None: async def api(methname, **kwargs) -> dict: + """Make (proxy) through an api call by name and return its result. + """ async with get_client() as client: meth = getattr(client.api, methname, None) if meth is None: log.error(f"No api method `{methname}` could be found?") - else: - arg = kwargs.get('arg') - if arg: - return await meth(arg) - else: - return await meth(**kwargs) + return + elif not kwargs: + # verify kwargs requirements are met + sig = inspect.signature(meth) + if sig.parameters: + log.error( + f"Argument(s) are required by the `{methname}` method: " + f"{tuple(sig.parameters.keys())}") + return + + return await meth(**kwargs) From 42ec8330f16558792a16cbc5bf0527b6c3bd1ef1 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sat, 27 Jan 2018 01:52:24 -0500 Subject: [PATCH 11/12] Explain the mess so far --- README.rst | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5b2b98f8..42f0f2cc 100644 --- a/README.rst +++ b/README.rst @@ -1,3 +1,21 @@ piker ------ -Destroy all suits +Anti-fragile trading gear for hackers, scientists, quants and underpants warriors. + + +Install +******* +``piker`` is currently under heavy alpha development and as such should +be cloned from this repo and hacked on directly. + +If you insist on trying to install it (which should work) please do it +from this GitHub repository:: + + pip install git+git://github.com/pikers/piker.git + + +Tech +**** +``piker`` is an attempt at a pro-grade, next-gen open source toolset +for trading and financial analysis. As such, it tries to use as much +cutting edge tech as possible including Python 3.6+ and ``trio``. From 797efedf6a1d1a5de4e75b08904333136cdec664 Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Mon, 29 Jan 2018 12:45:48 -0500 Subject: [PATCH 12/12] Add quote polling; pseudo-streaming Add a ``poll_tickers`` coro which can be used to "stream" quotes at a requested rate. Expose through a cli subcommand `piker stream`. Drop the `pikerd` command for now. --- piker/brokers/cli.py | 25 +++++++++++++++---------- piker/brokers/questrade.py | 32 +++++++++++++++++++++++++++++++- setup.py | 1 - 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/piker/brokers/cli.py b/piker/brokers/cli.py index 11678f7f..b5501633 100644 --- a/piker/brokers/cli.py +++ b/piker/brokers/cli.py @@ -1,7 +1,6 @@ """ Console interface to broker client/daemons. """ -import json from functools import partial from importlib import import_module @@ -23,15 +22,6 @@ def run(main, loglevel='info'): log.debug("Exiting piker") -@click.command() -@click.option('--broker', default='questrade', help='Broker backend to use') -@click.option('--loglevel', '-l', default='info', help='Logging level') -def pikerd(broker, loglevel): - # import broker module daemon entry point - brokermod = import_module('.' + broker, 'piker.brokers') - run(brokermod.serve_forever, loglevel) - - @click.group() def cli(): pass @@ -59,3 +49,18 @@ def api(meth, kwargs, loglevel, broker): data = run(partial(brokermod.api, meth, **_kwargs), loglevel=loglevel) if data: click.echo(colorize_json(data)) + + +@cli.command() +@click.option('--broker', default='questrade', help='Broker backend to use') +@click.option('--loglevel', '-l', default='info', help='Logging level') +@click.argument('tickers', nargs=-1) +def stream(broker, loglevel, tickers): + # import broker module daemon entry point + bm = import_module('.' + broker, 'piker.brokers') + run( + partial(bm.serve_forever, [ + partial(bm.poll_tickers, tickers=tickers) + ]), + loglevel + ) diff --git a/piker/brokers/questrade.py b/piker/brokers/questrade.py index f2661a73..3c30efea 100644 --- a/piker/brokers/questrade.py +++ b/piker/brokers/questrade.py @@ -140,6 +140,17 @@ class Client: self._prep_sess() return self.access_data + async def tickers2ids(self, tickers): + """Helper routine that take a sequence of ticker symbols and returns + their corresponding QT symbol ids. + """ + data = await self.api.symbols(names=','.join(tickers)) + symbols2ids = {} + for ticker, symbol in zip(tickers, data['symbols']): + symbols2ids[symbol['symbol']] = symbol['symbolId'] + + return symbols2ids + class API: """Questrade API at its finest. @@ -234,15 +245,34 @@ async def get_client() -> Client: write_conf(client) -async def serve_forever() -> None: +async def serve_forever(tasks) -> None: """Start up a client and serve until terminated. """ async with get_client() as client: # pretty sure this doesn't work # await client._revoke_auth_token() + async with trio.open_nursery() as nursery: + # launch token manager nursery.start_soon(token_refresher, client) + # launch children + for task in tasks: + nursery.start_soon(task, client) + + +async def poll_tickers(client, tickers, rate=2): + """Auto-poll snap quotes for a sequence of tickers at the given ``rate`` + per second. + """ + t2ids = await client.tickers2ids(tickers) + sleeptime = 1. / rate + ids = ','.join(map(str, t2ids.values())) + + while True: # use an event here to trigger exit? + quote_data = await client.api.quotes(ids=ids) + await trio.sleep(sleeptime) + async def api(methname, **kwargs) -> dict: """Make (proxy) through an api call by name and return its result. diff --git a/setup.py b/setup.py index f5edd117..59974c6e 100755 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ setup( ], entry_points={ 'console_scripts': [ - 'pikerd = piker.brokers.cli:pikerd', 'piker = piker.brokers.cli:cli', ] },