From cf1f4bed758e36d46e751a365f8512b4eb28dbae Mon Sep 17 00:00:00 2001 From: Tyler Goodlet Date: Sun, 25 Jun 2023 17:21:15 -0400 Subject: [PATCH] Move `.accounting` related config loaders to subpkg Like you'd think: - `load_ledger()` -> ._ledger - `load_accounrt()` -> ._pos Also fixup the old `load_pps_from_ledger()` and expose it from a new `.accounting.cli.disect` cli cmd for trying to figure out why pp calcs are totally mucked on stupid ib.. --- piker/accounting/_ledger.py | 48 ++++++++++- piker/accounting/_pos.py | 167 +++++++++++++++++++++++++++--------- piker/accounting/cli.py | 43 ++++++++-- piker/config.py | 93 -------------------- piker/log.py | 5 +- 5 files changed, 213 insertions(+), 143 deletions(-) diff --git a/piker/accounting/_ledger.py b/piker/accounting/_ledger.py index 5107f2bb..04ee04b7 100644 --- a/piker/accounting/_ledger.py +++ b/piker/accounting/_ledger.py @@ -123,6 +123,11 @@ class TransactionLedger(UserDict): self, t: Transaction, ) -> None: + ''' + Given an input `Transaction`, cast to `dict` and update + from it's transaction id. + + ''' self.data[t.tid] = t.to_dict() def iter_trans( @@ -259,6 +264,45 @@ def iter_by_dt( yield tid, data +def load_ledger( + brokername: str, + acctid: str, + +) -> tuple[dict, Path]: + ''' + Load a ledger (TOML) file from user's config directory: + $CONFIG_DIR/accounting/ledgers/trades__.toml + + Return its `dict`-content and file path. + + ''' + import time + try: + import tomllib + except ModuleNotFoundError: + import tomli as tomllib + + ldir: Path = config._config_dir / 'accounting' / 'ledgers' + if not ldir.is_dir(): + ldir.mkdir() + + fname = f'trades_{brokername}_{acctid}.toml' + fpath: Path = ldir / fname + + if not fpath.is_file(): + log.info( + f'Creating new local trades ledger: {fpath}' + ) + fpath.touch() + + with fpath.open(mode='rb') as cf: + start = time.time() + ledger_dict = tomllib.load(cf) + log.debug(f'Ledger load took {time.time() - start}s') + + return ledger_dict, fpath + + @cm def open_trade_ledger( broker: str, @@ -267,7 +311,7 @@ def open_trade_ledger( # default is to sort by detected datetime-ish field tx_sort: Callable = iter_by_dt, -) -> Generator[dict, None, None]: +) -> Generator[TransactionLedger, None, None]: ''' Indempotently create and read in a trade log file from the ``/ledgers/`` directory. @@ -277,7 +321,7 @@ def open_trade_ledger( name as defined in the user's ``brokers.toml`` config. ''' - ledger_dict, fpath = config.load_ledger(broker, account) + ledger_dict, fpath = load_ledger(broker, account) cpy = ledger_dict.copy() ledger = TransactionLedger( ledger_dict=cpy, diff --git a/piker/accounting/_pos.py b/piker/accounting/_pos.py index 3af0eeef..f50040cb 100644 --- a/piker/accounting/_pos.py +++ b/piker/accounting/_pos.py @@ -42,6 +42,7 @@ from ._ledger import ( Transaction, iter_by_dt, open_trade_ledger, + TransactionLedger, ) from ._mktinfo import ( MktPair, @@ -49,7 +50,6 @@ from ._mktinfo import ( unpack_fqme, ) from .. import config -from ..brokers import get_brokermod from ..clearing._messages import ( BrokerdPosition, Status, @@ -327,7 +327,8 @@ class Position(Struct): entry: dict[str, Any] for (tid, entry) in self.iter_clears(): clear_size = entry['size'] - clear_price = entry['price'] + clear_price: str | float = entry['price'] + is_clear: bool = not isinstance(clear_price, str) last_accum_size = asize_h[-1] if asize_h else 0 accum_size = last_accum_size + clear_size @@ -340,9 +341,18 @@ class Position(Struct): asize_h.append(0) continue - if accum_size == 0: - ppu_h.append(0) - asize_h.append(0) + # on transfers we normally write some non-valid + # price since withdrawal to another account/wallet + # has nothing to do with inter-asset-market prices. + # TODO: this should be better handled via a `type: 'tx'` + # field as per existing issue surrounding all this: + # https://github.com/pikers/piker/issues/510 + if isinstance(clear_price, str): + # TODO: we can't necessarily have this commit to + # the overall pos size since we also need to + # include other positions contributions to this + # balance or we might end up with a -ve balance for + # the position.. continue # test if the pp somehow went "passed" a net zero size state @@ -375,7 +385,10 @@ class Position(Struct): # abs_clear_size = abs(clear_size) abs_new_size = abs(accum_size) - if abs_diff > 0: + if ( + abs_diff > 0 + and is_clear + ): cost_basis = ( # cost basis for this clear @@ -397,6 +410,12 @@ class Position(Struct): asize_h.append(accum_size) else: + # TODO: for PPU we should probably handle txs out + # (aka withdrawals) similarly by simply not having + # them contrib to the running PPU calc and only + # when the next entry clear comes in (which will + # then have a higher weighting on the PPU). + # on "exit" clears from a given direction, # only the size changes not the price-per-unit # need to be updated since the ppu remains constant @@ -734,48 +753,63 @@ class PpTable(Struct): ) -def load_pps_from_ledger( - +def load_account( brokername: str, - acctname: str, + acctid: str, - # post normalization filter on ledger entries to be processed - filter_by: list[dict] | None = None, - -) -> tuple[ - dict[str, Transaction], - dict[str, Position], -]: +) -> tuple[dict, Path]: ''' - Open a ledger file by broker name and account and read in and - process any trade records into our normalized ``Transaction`` form - and then update the equivalent ``Pptable`` and deliver the two - bs_mktid-mapped dict-sets of the transactions and pps. + Load a accounting (with positions) file from + $CONFIG_DIR/accounting/account...toml + + Where normally $CONFIG_DIR = ~/.config/piker/ + and we implicitly create a accounting subdir which should + normally be linked to a git repo managed by the user B) ''' - with ( - open_trade_ledger(brokername, acctname) as ledger, - open_pps(brokername, acctname) as table, - ): - if not ledger: - # null case, no ledger file with content - return {} + legacy_fn: str = f'pps.{brokername}.{acctid}.toml' + fn: str = f'account.{brokername}.{acctid}.toml' - mod = get_brokermod(brokername) - src_records: dict[str, Transaction] = mod.norm_trade_records(ledger) + dirpath: Path = config._config_dir / 'accounting' + if not dirpath.is_dir(): + dirpath.mkdir() - if filter_by: - records = {} - bs_mktids = set(filter_by) - for tid, r in src_records.items(): - if r.bs_mktid in bs_mktids: - records[tid] = r - else: - records = src_records + conf, path = config.load( + path=dirpath / fn, + decode=tomlkit.parse, + touch_if_dne=True, + ) - updated = table.update_from_trans(records) + if not conf: + legacypath = dirpath / legacy_fn + log.warning( + f'Your account file is using the legacy `pps.` prefix..\n' + f'Rewriting contents to new name -> {path}\n' + 'Please delete the old file!\n' + f'|-> {legacypath}\n' + ) + if legacypath.is_file(): + legacy_config, _ = config.load( + path=legacypath, - return records, updated + # TODO: move to tomlkit: + # - needs to be fixed to support bidict? + # https://github.com/sdispater/tomlkit/issues/289 + # - we need to use or fork's fix to do multiline array + # indenting. + decode=tomlkit.parse, + ) + conf.update(legacy_config) + + # XXX: override the presumably previously non-existant + # file with legacy's contents. + config.write( + conf, + path=path, + fail_empty=False, + ) + + return conf, path @cm @@ -792,7 +826,7 @@ def open_pps( ''' conf: dict conf_path: Path - conf, conf_path = config.load_account(brokername, acctid) + conf, conf_path = load_account(brokername, acctid) if brokername in conf: log.warning( @@ -927,3 +961,56 @@ def open_pps( finally: if write_on_exit: table.write_config() + + +def load_pps_from_ledger( + + brokername: str, + acctname: str, + + # post normalization filter on ledger entries to be processed + filter_by_ids: list[str] | None = None, + +) -> tuple[ + dict[str, Transaction], + PpTable, +]: + ''' + Open a ledger file by broker name and account and read in and + process any trade records into our normalized ``Transaction`` form + and then update the equivalent ``Pptable`` and deliver the two + bs_mktid-mapped dict-sets of the transactions and pps. + + ''' + ledger: TransactionLedger + table: PpTable + with ( + open_trade_ledger(brokername, acctname) as ledger, + open_pps(brokername, acctname) as table, + ): + if not ledger: + # null case, no ledger file with content + return {} + + from ..brokers import get_brokermod + mod = get_brokermod(brokername) + src_records: dict[str, Transaction] = mod.norm_trade_records( + ledger + ) + + if not filter_by_ids: + # records = src_records + records = ledger + + else: + records = {} + bs_mktids = set(map(str, filter_by_ids)) + + # for tid, recdict in ledger.items(): + for tid, r in src_records.items(): + if r.bs_mktid in bs_mktids: + records[tid] = r.to_dict() + + # updated = table.update_from_trans(records) + + return records, table diff --git a/piker/accounting/cli.py b/piker/accounting/cli.py index 75798f3f..c184614c 100644 --- a/piker/accounting/cli.py +++ b/piker/accounting/cli.py @@ -18,6 +18,7 @@ CLI front end for trades ledger and position tracking management. ''' +from __future__ import annotations from rich.console import Console from rich.markdown import Markdown import tractor @@ -29,9 +30,18 @@ from ..service import ( open_piker_runtime, ) from ..clearing._messages import BrokerdPosition -from ..config import load_ledger from ..calc import humanize from ..brokers._daemon import broker_init +from ._ledger import ( + load_ledger, + # open_trade_ledger, + TransactionLedger, +) +from ._pos import ( + PpTable, + load_pps_from_ledger, + # load_account, +) ledger = typer.Typer() @@ -39,7 +49,7 @@ ledger = typer.Typer() def unpack_fqan( fully_qualified_account_name: str, - console: Console | None, + console: Console | None = None, ) -> tuple | bool: try: brokername, account = fully_qualified_account_name.split('.') @@ -225,7 +235,8 @@ def sync( @ledger.command() def disect( - fully_qualified_account_name: str, + # "fully_qualified_account_name" + fqan: str, bs_mktid: int, # for ib pdb: bool = False, @@ -235,10 +246,28 @@ def disect( ), ): pair: tuple[str, str] - if not (pair := unpack_fqan( - fully_qualified_account_name, - )): - return + if not (pair := unpack_fqan(fqan)): + raise ValueError('{fqan} malformed!?') + + brokername, account = pair + + ledger: TransactionLedger + table: PpTable + records, table = load_pps_from_ledger( + brokername, + account, + # filter_by_id = {568549458}, + filter_by_ids={bs_mktid}, + ) + breakpoint() + # tractor.pause_from_sync() + # with open_trade_ledger( + # brokername, + # account, + # ) as ledger: + # for tid, rec in ledger.items(): + # bs_mktid: str = rec['bs_mktid'] + if __name__ == "__main__": diff --git a/piker/config.py b/piker/config.py index 3bb026d5..80f7b1d1 100644 --- a/piker/config.py +++ b/piker/config.py @@ -22,7 +22,6 @@ import platform import sys import os import shutil -import time from typing import ( Callable, MutableMapping, @@ -310,98 +309,6 @@ def load( return config, path -def load_account( - brokername: str, - acctid: str, - -) -> tuple[dict, Path]: - ''' - Load a accounting (with positions) file from - $CONFIG_DIR/accounting/account...toml - - Where normally $CONFIG_DIR = ~/.config/piker/ - and we implicitly create a accounting subdir which should - normally be linked to a git repo managed by the user B) - - ''' - legacy_fn: str = f'pps.{brokername}.{acctid}.toml' - fn: str = f'account.{brokername}.{acctid}.toml' - - dirpath: Path = _config_dir / 'accounting' - if not dirpath.is_dir(): - dirpath.mkdir() - - config, path = load( - path=dirpath / fn, - decode=tomlkit.parse, - touch_if_dne=True, - ) - - if not config: - legacypath = dirpath / legacy_fn - log.warning( - f'Your account file is using the legacy `pps.` prefix..\n' - f'Rewriting contents to new name -> {path}\n' - 'Please delete the old file!\n' - f'|-> {legacypath}\n' - ) - if legacypath.is_file(): - legacy_config, _ = load( - path=legacypath, - - # TODO: move to tomlkit: - # - needs to be fixed to support bidict? - # https://github.com/sdispater/tomlkit/issues/289 - # - we need to use or fork's fix to do multiline array - # indenting. - decode=tomlkit.parse, - ) - config.update(legacy_config) - - # XXX: override the presumably previously non-existant - # file with legacy's contents. - write( - config, - path=path, - fail_empty=False, - ) - - return config, path - - -def load_ledger( - brokername: str, - acctid: str, - -) -> tuple[dict, Path]: - ''' - Load a ledger (TOML) file from user's config directory: - $CONFIG_DIR/accounting/ledgers/trades__.toml - - Return its `dict`-content and file path. - - ''' - ldir: Path = _config_dir / 'accounting' / 'ledgers' - if not ldir.is_dir(): - ldir.mkdir() - - fname = f'trades_{brokername}_{acctid}.toml' - fpath: Path = ldir / fname - - if not fpath.is_file(): - log.info( - f'Creating new local trades ledger: {fpath}' - ) - fpath.touch() - - with fpath.open(mode='rb') as cf: - start = time.time() - ledger_dict = tomllib.load(cf) - log.debug(f'Ledger load took {time.time() - start}s') - - return ledger_dict, fpath - - def write( config: dict, # toml config as dict diff --git a/piker/log.py b/piker/log.py index a36beec0..56776e1e 100644 --- a/piker/log.py +++ b/piker/log.py @@ -40,7 +40,10 @@ def get_logger( Return the package log or a sub-log for `name` if provided. ''' - return tractor.log.get_logger(name=name, _root_name=_proj_name) + return tractor.log.get_logger( + name=name, + _root_name=_proj_name, + ) def get_console_log(