.tsp: report non-positive timestamp steps
- keep positive missing-history gaps distinct from ordering faults - return duplicate and reversed timestamp rows as structured data - warn before normalization removes the original ordering evidence Prompt-IO: ai/prompt-io/opencode/20260722T024133Z_3a4f6737_prompt_io.md (this patch was generated in some part by `opencode` using `gpt-5.6-sol` (`openai`))backfiller_deep_fixes
parent
6913e80d1a
commit
c60a5a45f3
|
|
@ -0,0 +1,39 @@
|
|||
---
|
||||
model: gpt-5.6-sol
|
||||
provider: openai
|
||||
service: opencode
|
||||
session: ses_0799212ebffe42arY96czXn89F
|
||||
timestamp: 2026-07-22T02:41:33Z
|
||||
git_ref: 3a4f6737
|
||||
scope: code
|
||||
substantive: true
|
||||
raw_file: 20260722T024133Z_3a4f6737_prompt_io.raw.md
|
||||
---
|
||||
|
||||
## Prompt
|
||||
|
||||
The user challenged the analysis commit message's claim that only
|
||||
positive sample-step gaps are detected and asked whether out-of-order
|
||||
timestamps were simply ignored. After confirming the missing separate
|
||||
reporting path, the user instructed the agent to implement the fix.
|
||||
|
||||
## Response summary
|
||||
|
||||
Separated missing-history gaps from timestamp-ordering errors without
|
||||
changing the existing `dedupe()` return shape. Added a structured
|
||||
detector and made normalization warn before sorting removes evidence of
|
||||
duplicates or reversed timestamps.
|
||||
|
||||
## Files changed
|
||||
|
||||
- `piker/tsp/_anal.py` - ordering detector and dedupe reporting
|
||||
- `piker/tsp/__init__.py` - public detector export
|
||||
- `tests/test_tsp_analysis.py` - exact ordering-error regression
|
||||
- `ai/prompt-io/opencode/20260722T024133Z_3a4f6737_prompt_io.raw.md`
|
||||
- unedited response record
|
||||
- `ai/prompt-io/opencode/20260722T024133Z_3a4f6737_prompt_io.md`
|
||||
- provenance metadata and response summary
|
||||
|
||||
## Human edits
|
||||
|
||||
None - generated changes have not yet been edited by the human.
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
---
|
||||
model: gpt-5.6-sol
|
||||
provider: openai
|
||||
service: opencode
|
||||
timestamp: 2026-07-22T02:41:33Z
|
||||
git_ref: 3a4f6737
|
||||
diff_cmd: git diff HEAD~1..HEAD
|
||||
---
|
||||
|
||||
> `git diff HEAD~1..HEAD -- piker/tsp/_anal.py`
|
||||
|
||||
Added `detect_time_ordering_errors()` to return non-positive timestamp
|
||||
steps as structured rows. `dedupe()` now warns with those rows before
|
||||
sorting and duplicate removal erase the original ordering evidence.
|
||||
|
||||
> `git diff HEAD~1..HEAD -- piker/tsp/__init__.py`
|
||||
|
||||
Exported the ordering detector through the public TSP package.
|
||||
|
||||
> `git diff HEAD~1..HEAD -- tests/test_tsp_analysis.py`
|
||||
|
||||
Changed the negative-delta regression to verify exact predecessor,
|
||||
timestamp, and delta values, confirm exclusion from missing-history
|
||||
gaps, and require explicit normalization reporting.
|
||||
|
||||
Verification generated with the patch:
|
||||
|
||||
- `tests/test_tsp_analysis.py`: 11 passed
|
||||
- Ruff: passed
|
||||
- `git diff --check`: passed
|
||||
|
|
@ -34,6 +34,7 @@ from ._anal import (
|
|||
# `polars` specific
|
||||
dedupe as dedupe,
|
||||
detect_time_gaps as detect_time_gaps,
|
||||
detect_time_ordering_errors as detect_time_ordering_errors,
|
||||
pl2np as pl2np,
|
||||
np2pl as np2pl,
|
||||
|
||||
|
|
|
|||
|
|
@ -512,6 +512,22 @@ def detect_time_gaps(
|
|||
)
|
||||
|
||||
|
||||
def detect_time_ordering_errors(
|
||||
w_dts: pl.DataFrame,
|
||||
|
||||
) -> pl.DataFrame:
|
||||
'''
|
||||
Return rows whose timestamp does not follow its predecessor.
|
||||
|
||||
Zero deltas identify duplicate timestamps and negative deltas
|
||||
identify out-of-order rows. Neither is a missing-history gap.
|
||||
|
||||
'''
|
||||
return w_dts.filter(
|
||||
pl.col('s_diff') <= 0
|
||||
)
|
||||
|
||||
|
||||
def detect_price_gaps(
|
||||
df: pl.DataFrame,
|
||||
gt_multiplier: float = 2.,
|
||||
|
|
@ -574,6 +590,13 @@ def dedupe(
|
|||
|
||||
'''
|
||||
wdts: pl.DataFrame = with_dts(src_df)
|
||||
ordering_errors = detect_time_ordering_errors(wdts)
|
||||
if not ordering_errors.is_empty():
|
||||
log.warning(
|
||||
f'Found {ordering_errors.height} non-positive '
|
||||
f'timestamp step(s) before normalization:\n'
|
||||
f'{ordering_errors}'
|
||||
)
|
||||
|
||||
# remove duplicated datetime samples/sections
|
||||
deduped: pl.DataFrame = src_df.unique(
|
||||
|
|
@ -585,9 +608,6 @@ def dedupe(
|
|||
# maybe sort on any time field
|
||||
if sort:
|
||||
deduped = deduped.sort(by='time')
|
||||
# TODO: detect out-of-order segments which were corrected!
|
||||
# -[ ] report in log msg
|
||||
# -[ ] possibly return segment sections which were moved?
|
||||
|
||||
deduped = with_dts(deduped)
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from piker.tsp import _anal
|
|||
from piker.tsp._anal import (
|
||||
dedupe,
|
||||
detect_time_gaps,
|
||||
detect_time_ordering_errors,
|
||||
get_null_segs,
|
||||
iter_null_segs,
|
||||
np2pl,
|
||||
|
|
@ -102,19 +103,30 @@ def test_dedupe_recomputes_dts_after_sort() -> None:
|
|||
).is_empty()
|
||||
|
||||
|
||||
def test_negative_delta_is_not_missing_history() -> None:
|
||||
def test_negative_delta_is_an_ordering_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
'''
|
||||
Reversed timestamps are ordering faults, not gaps.
|
||||
Reversed timestamps are reported as ordering faults, not gaps.
|
||||
|
||||
'''
|
||||
wdts: pl.DataFrame = with_dts(
|
||||
np2pl(mk_ohlcv([120, 60]))
|
||||
)
|
||||
src = np2pl(mk_ohlcv([120, 60]))
|
||||
wdts: pl.DataFrame = with_dts(src)
|
||||
errors = detect_time_ordering_errors(wdts)
|
||||
|
||||
assert detect_time_gaps(
|
||||
wdts,
|
||||
expect_period=30,
|
||||
).is_empty()
|
||||
assert errors['time_prev'].to_list() == [120]
|
||||
assert errors['time'].to_list() == [60]
|
||||
assert errors['s_diff'].to_list() == [-60]
|
||||
|
||||
warnings: list[str] = []
|
||||
monkeypatch.setattr(_anal.log, 'warning', warnings.append)
|
||||
dedupe(src)
|
||||
assert len(warnings) == 1
|
||||
assert 'non-positive timestamp step' in warnings[0]
|
||||
|
||||
|
||||
def test_single_null_row_is_a_segment() -> None:
|
||||
|
|
|
|||
Loading…
Reference in New Issue