.tsp: report non-positive timestamp steps
- keep positive missing-history gaps distinct from ordering faults - return duplicate and reversed timestamp rows as structured data - warn before normalization removes the original ordering evidence Prompt-IO: ai/prompt-io/opencode/20260722T024133Z_3a4f6737_prompt_io.md (this patch was generated in some part by `opencode` using `gpt-5.6-sol` (`openai`))backfiller_deep_fixes
parent
6913e80d1a
commit
c60a5a45f3
|
|
@ -0,0 +1,39 @@
|
||||||
|
---
|
||||||
|
model: gpt-5.6-sol
|
||||||
|
provider: openai
|
||||||
|
service: opencode
|
||||||
|
session: ses_0799212ebffe42arY96czXn89F
|
||||||
|
timestamp: 2026-07-22T02:41:33Z
|
||||||
|
git_ref: 3a4f6737
|
||||||
|
scope: code
|
||||||
|
substantive: true
|
||||||
|
raw_file: 20260722T024133Z_3a4f6737_prompt_io.raw.md
|
||||||
|
---
|
||||||
|
|
||||||
|
## Prompt
|
||||||
|
|
||||||
|
The user challenged the analysis commit message's claim that only
|
||||||
|
positive sample-step gaps are detected and asked whether out-of-order
|
||||||
|
timestamps were simply ignored. After confirming the missing separate
|
||||||
|
reporting path, the user instructed the agent to implement the fix.
|
||||||
|
|
||||||
|
## Response summary
|
||||||
|
|
||||||
|
Separated missing-history gaps from timestamp-ordering errors without
|
||||||
|
changing the existing `dedupe()` return shape. Added a structured
|
||||||
|
detector and made normalization warn before sorting removes evidence of
|
||||||
|
duplicates or reversed timestamps.
|
||||||
|
|
||||||
|
## Files changed
|
||||||
|
|
||||||
|
- `piker/tsp/_anal.py` - ordering detector and dedupe reporting
|
||||||
|
- `piker/tsp/__init__.py` - public detector export
|
||||||
|
- `tests/test_tsp_analysis.py` - exact ordering-error regression
|
||||||
|
- `ai/prompt-io/opencode/20260722T024133Z_3a4f6737_prompt_io.raw.md`
|
||||||
|
- unedited response record
|
||||||
|
- `ai/prompt-io/opencode/20260722T024133Z_3a4f6737_prompt_io.md`
|
||||||
|
- provenance metadata and response summary
|
||||||
|
|
||||||
|
## Human edits
|
||||||
|
|
||||||
|
None - generated changes have not yet been edited by the human.
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
---
|
||||||
|
model: gpt-5.6-sol
|
||||||
|
provider: openai
|
||||||
|
service: opencode
|
||||||
|
timestamp: 2026-07-22T02:41:33Z
|
||||||
|
git_ref: 3a4f6737
|
||||||
|
diff_cmd: git diff HEAD~1..HEAD
|
||||||
|
---
|
||||||
|
|
||||||
|
> `git diff HEAD~1..HEAD -- piker/tsp/_anal.py`
|
||||||
|
|
||||||
|
Added `detect_time_ordering_errors()` to return non-positive timestamp
|
||||||
|
steps as structured rows. `dedupe()` now warns with those rows before
|
||||||
|
sorting and duplicate removal erase the original ordering evidence.
|
||||||
|
|
||||||
|
> `git diff HEAD~1..HEAD -- piker/tsp/__init__.py`
|
||||||
|
|
||||||
|
Exported the ordering detector through the public TSP package.
|
||||||
|
|
||||||
|
> `git diff HEAD~1..HEAD -- tests/test_tsp_analysis.py`
|
||||||
|
|
||||||
|
Changed the negative-delta regression to verify exact predecessor,
|
||||||
|
timestamp, and delta values, confirm exclusion from missing-history
|
||||||
|
gaps, and require explicit normalization reporting.
|
||||||
|
|
||||||
|
Verification generated with the patch:
|
||||||
|
|
||||||
|
- `tests/test_tsp_analysis.py`: 11 passed
|
||||||
|
- Ruff: passed
|
||||||
|
- `git diff --check`: passed
|
||||||
|
|
@ -34,6 +34,7 @@ from ._anal import (
|
||||||
# `polars` specific
|
# `polars` specific
|
||||||
dedupe as dedupe,
|
dedupe as dedupe,
|
||||||
detect_time_gaps as detect_time_gaps,
|
detect_time_gaps as detect_time_gaps,
|
||||||
|
detect_time_ordering_errors as detect_time_ordering_errors,
|
||||||
pl2np as pl2np,
|
pl2np as pl2np,
|
||||||
np2pl as np2pl,
|
np2pl as np2pl,
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -512,6 +512,22 @@ def detect_time_gaps(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def detect_time_ordering_errors(
|
||||||
|
w_dts: pl.DataFrame,
|
||||||
|
|
||||||
|
) -> pl.DataFrame:
|
||||||
|
'''
|
||||||
|
Return rows whose timestamp does not follow its predecessor.
|
||||||
|
|
||||||
|
Zero deltas identify duplicate timestamps and negative deltas
|
||||||
|
identify out-of-order rows. Neither is a missing-history gap.
|
||||||
|
|
||||||
|
'''
|
||||||
|
return w_dts.filter(
|
||||||
|
pl.col('s_diff') <= 0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def detect_price_gaps(
|
def detect_price_gaps(
|
||||||
df: pl.DataFrame,
|
df: pl.DataFrame,
|
||||||
gt_multiplier: float = 2.,
|
gt_multiplier: float = 2.,
|
||||||
|
|
@ -574,6 +590,13 @@ def dedupe(
|
||||||
|
|
||||||
'''
|
'''
|
||||||
wdts: pl.DataFrame = with_dts(src_df)
|
wdts: pl.DataFrame = with_dts(src_df)
|
||||||
|
ordering_errors = detect_time_ordering_errors(wdts)
|
||||||
|
if not ordering_errors.is_empty():
|
||||||
|
log.warning(
|
||||||
|
f'Found {ordering_errors.height} non-positive '
|
||||||
|
f'timestamp step(s) before normalization:\n'
|
||||||
|
f'{ordering_errors}'
|
||||||
|
)
|
||||||
|
|
||||||
# remove duplicated datetime samples/sections
|
# remove duplicated datetime samples/sections
|
||||||
deduped: pl.DataFrame = src_df.unique(
|
deduped: pl.DataFrame = src_df.unique(
|
||||||
|
|
@ -585,9 +608,6 @@ def dedupe(
|
||||||
# maybe sort on any time field
|
# maybe sort on any time field
|
||||||
if sort:
|
if sort:
|
||||||
deduped = deduped.sort(by='time')
|
deduped = deduped.sort(by='time')
|
||||||
# TODO: detect out-of-order segments which were corrected!
|
|
||||||
# -[ ] report in log msg
|
|
||||||
# -[ ] possibly return segment sections which were moved?
|
|
||||||
|
|
||||||
deduped = with_dts(deduped)
|
deduped = with_dts(deduped)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@ from piker.tsp import _anal
|
||||||
from piker.tsp._anal import (
|
from piker.tsp._anal import (
|
||||||
dedupe,
|
dedupe,
|
||||||
detect_time_gaps,
|
detect_time_gaps,
|
||||||
|
detect_time_ordering_errors,
|
||||||
get_null_segs,
|
get_null_segs,
|
||||||
iter_null_segs,
|
iter_null_segs,
|
||||||
np2pl,
|
np2pl,
|
||||||
|
|
@ -102,19 +103,30 @@ def test_dedupe_recomputes_dts_after_sort() -> None:
|
||||||
).is_empty()
|
).is_empty()
|
||||||
|
|
||||||
|
|
||||||
def test_negative_delta_is_not_missing_history() -> None:
|
def test_negative_delta_is_an_ordering_error(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
) -> None:
|
||||||
'''
|
'''
|
||||||
Reversed timestamps are ordering faults, not gaps.
|
Reversed timestamps are reported as ordering faults, not gaps.
|
||||||
|
|
||||||
'''
|
'''
|
||||||
wdts: pl.DataFrame = with_dts(
|
src = np2pl(mk_ohlcv([120, 60]))
|
||||||
np2pl(mk_ohlcv([120, 60]))
|
wdts: pl.DataFrame = with_dts(src)
|
||||||
)
|
errors = detect_time_ordering_errors(wdts)
|
||||||
|
|
||||||
assert detect_time_gaps(
|
assert detect_time_gaps(
|
||||||
wdts,
|
wdts,
|
||||||
expect_period=30,
|
expect_period=30,
|
||||||
).is_empty()
|
).is_empty()
|
||||||
|
assert errors['time_prev'].to_list() == [120]
|
||||||
|
assert errors['time'].to_list() == [60]
|
||||||
|
assert errors['s_diff'].to_list() == [-60]
|
||||||
|
|
||||||
|
warnings: list[str] = []
|
||||||
|
monkeypatch.setattr(_anal.log, 'warning', warnings.append)
|
||||||
|
dedupe(src)
|
||||||
|
assert len(warnings) == 1
|
||||||
|
assert 'non-positive timestamp step' in warnings[0]
|
||||||
|
|
||||||
|
|
||||||
def test_single_null_row_is_a_segment() -> None:
|
def test_single_null_row_is_a_segment() -> None:
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue