From 4c3a51a32627d0f57b535b5ee9128c080a1ede4d Mon Sep 17 00:00:00 2001
From: Tyler Goodlet <jgbt@protonmail.com>
Date: Mon, 15 Sep 2025 18:29:19 -0400
Subject: [PATCH] ib-related: cope with invalid txn timestamps

That is inside embedded `.accounting.calc.dyn_parse_to_dt()` closure add
an optional `_invalid: list` param to where we can report
bad-timestamped records which we instead override and return as
`from_timestamp(0.)` (when the parser loop falls through) and report
later (in summary ) from the `.accounting.calc.iter_by_dt()` caller. Add
some logging and an optional debug block for future tracing.

NOTE, this commit was re-edited during a conflict between the orig
branches: `dev/binance_api_3.1` & `dev/alt_tpts_for_perf`.
---
 piker/accounting/calc.py | 82 +++++++++++++++++++++++++++++-----------
 1 file changed, 60 insertions(+), 22 deletions(-)

diff --git a/piker/accounting/calc.py b/piker/accounting/calc.py
index ff12c055..64765e76 100644
--- a/piker/accounting/calc.py
+++ b/piker/accounting/calc.py
@@ -22,7 +22,9 @@ you know when you're losing money (if possible) XD
 from __future__ import annotations
 from collections.abc import ValuesView
 from contextlib import contextmanager as cm
+from functools import partial
 from math import copysign
+from pprint import pformat
 from typing import (
     Any,
     Callable,
@@ -37,12 +39,16 @@ from pendulum import (
     parse,
 )
 
+from ..log import get_logger
+
 if TYPE_CHECKING:
     from ._ledger import (
         Transaction,
         TransactionLedger,
     )
 
+log = get_logger(__name__)
+
 
 def ppu(
     clears: Iterator[Transaction],
@@ -238,6 +244,9 @@ def iter_by_dt(
 
     def dyn_parse_to_dt(
         tx: tuple[str, dict[str, Any]] | Transaction,
+
+        debug: bool = False,
+        _invalid: list|None = None,
     ) -> DateTime:
 
         # handle `.items()` inputs
@@ -250,52 +259,81 @@ def iter_by_dt(
         # get best parser for this record..
         for k in parsers:
             if (
-                isdict and k in tx
+                (v := getattr(tx, k, None))
                 or
-                getattr(tx, k, None)
+                (
+                    isdict
+                    and
+                    (v := tx.get(k))
+                )
             ):
-                v = (
-                    tx[k] if isdict
-                    else tx.dt
-                )
-                assert v is not None, (
-                    f'No valid value for `{k}`!?'
-                )
-
                 # only call parser on the value if not None from
                 # the `parsers` table above (when NOT using
                 # `.get()`), otherwise pass through the value and
                 # sort on it directly
                 if (
                     not isinstance(v, DateTime)
-                    and (parser := parsers.get(k))
+                    and
+                    (parser := parsers.get(k))
                 ):
-                    return parser(v)
+                    ret = parser(v)
                 else:
-                    return v
+                    ret = v
 
+                return ret
+
+            else:
+                log.debug(
+                    f'Parser-field not found in txn\n'
+                    f'\n'
+                    f'parser-field: {k!r}\n'
+                    f'txn: {tx!r}\n'
+                    f'\n'
+                    f'Trying next..\n'
+                )
+                continue
+
+        # XXX: should never get here..
         else:
-            # TODO: move to top?
-            from piker.log import get_logger
-            log = get_logger(__name__)
-
             # XXX: we should really never get here..
             # only if a ledger record has no expected sort(able)
             # field will we likely hit this.. like with ze IB.
             # if no sortable field just deliver epoch?
             log.warning(
                 'No (time) sortable field for TXN:\n'
-                f'{tx}\n'
+                f'{tx!r}\n'
             )
-            return from_timestamp(0)
-            # breakpoint()
+            if debug:
+                import tractor
+                with tractor.devx.maybe_open_crash_handler():
+                    raise ValueError(
+                        f'No supported time-field found in txn !?\n'
+                        f'\n'
+                        f'supported-time-fields: {parsers!r}\n'
+                        f'\n'
+                        f'txn: {tx!r}\n'
+                    )
 
+            if _invalid is not None:
+                _invalid.append(tx)
+                return from_timestamp(0.)
 
-    entry: tuple[str, dict] | Transaction
+    entry: tuple[str, dict]|Transaction
+    invalid: list = []
     for entry in sorted(
         records,
-        key=key or dyn_parse_to_dt,
+        key=key or partial(
+            dyn_parse_to_dt,
+            _invalid=invalid,
+        ),
     ):
+        if entry in invalid:
+            log.warning(
+                f'Ignoring txn w invalid timestamp ??\n'
+                f'{pformat(entry)}\n'
+            )
+            continue
+
         # NOTE the type sig above; either pairs or txns B)
         yield entry