653 lines
18 KiB
Python
653 lines
18 KiB
Python
# piker: trading gear for hackers
|
|
# Copyright (C) Tyler Goodlet (in stewardship for pikers)
|
|
|
|
'''
|
|
Read-only OHLCV persistence qualification.
|
|
|
|
'''
|
|
from datetime import (
|
|
UTC,
|
|
datetime,
|
|
)
|
|
from hashlib import sha256
|
|
import os
|
|
from pathlib import Path
|
|
import shutil
|
|
from stat import S_ISREG
|
|
from typing import BinaryIO
|
|
|
|
import numpy as np
|
|
import polars as pl
|
|
|
|
from piker.data import def_iohlcv_fields
|
|
|
|
|
|
AUDIT_SCHEMA: str = 'piker.nativedb.audit/v1'
|
|
_value_fields: tuple[str, ...] = (
|
|
'open',
|
|
'high',
|
|
'low',
|
|
'close',
|
|
'volume',
|
|
)
|
|
|
|
|
|
def _utc_str(timestamp: float|int|None) -> str|None:
|
|
'''
|
|
Render an epoch timestamp as deterministic UTC.
|
|
|
|
'''
|
|
if timestamp is None:
|
|
return None
|
|
try:
|
|
dt: datetime = datetime.fromtimestamp(timestamp, tz=UTC)
|
|
except (
|
|
OSError,
|
|
OverflowError,
|
|
ValueError,
|
|
):
|
|
return None
|
|
return dt.isoformat().replace('+00:00', 'Z')
|
|
|
|
|
|
def _number(value: float|int) -> float|int:
|
|
'''
|
|
Convert a NumPy number to a JSON-native scalar.
|
|
|
|
'''
|
|
if isinstance(value, (int, np.integer)):
|
|
return int(value)
|
|
value = float(value)
|
|
return int(value) if value.is_integer() else value
|
|
|
|
|
|
def _sha256(file: BinaryIO) -> str:
|
|
'''
|
|
Hash and rewind one already-open file snapshot.
|
|
|
|
'''
|
|
digest = sha256()
|
|
for chunk in iter(
|
|
lambda: file.read(1024 * 1024),
|
|
b'',
|
|
):
|
|
digest.update(chunk)
|
|
file.seek(0)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _numeric_counts(series: pl.Series) -> dict[str, int|None]:
|
|
'''
|
|
Count null and non-finite values without coercing text.
|
|
|
|
'''
|
|
null_count: int = series.null_count()
|
|
if not series.dtype.is_numeric():
|
|
return {
|
|
'null': null_count,
|
|
'nan': None,
|
|
'positive_infinity': None,
|
|
'negative_infinity': None,
|
|
'nonfinite': None,
|
|
}
|
|
|
|
numeric: pl.Series = series.cast(pl.Float64, strict=False)
|
|
nan_count: int = int(numeric.is_nan().sum() or 0)
|
|
pos_inf_count: int = int(
|
|
(numeric == float('inf')).sum() or 0
|
|
)
|
|
neg_inf_count: int = int(
|
|
(numeric == float('-inf')).sum() or 0
|
|
)
|
|
cast_null_count: int = max(
|
|
0,
|
|
numeric.null_count() - null_count,
|
|
)
|
|
return {
|
|
'null': null_count,
|
|
'nan': nan_count,
|
|
'positive_infinity': pos_inf_count,
|
|
'negative_infinity': neg_inf_count,
|
|
'nonfinite': (
|
|
null_count
|
|
+ nan_count
|
|
+ pos_inf_count
|
|
+ neg_inf_count
|
|
+ cast_null_count
|
|
),
|
|
}
|
|
|
|
|
|
def _numeric_values(
|
|
series: pl.Series,
|
|
) -> tuple[np.ndarray, np.ndarray]:
|
|
'''
|
|
Preserve integer precision and return an explicit valid mask.
|
|
|
|
'''
|
|
if series.dtype.is_integer():
|
|
valid: np.ndarray = series.is_not_null().to_numpy()
|
|
values: np.ndarray = series.fill_null(0).to_numpy()
|
|
return values, valid
|
|
|
|
values = series.to_numpy()
|
|
try:
|
|
valid = np.isfinite(values)
|
|
except TypeError:
|
|
values = series.cast(pl.Float64).to_numpy()
|
|
valid = np.isfinite(values)
|
|
return values, valid
|
|
|
|
|
|
def _empty_gaps(
|
|
period_s: int,
|
|
verifiable: bool = False,
|
|
) -> dict:
|
|
'''
|
|
Return the stable gap report shape for unauditable timestamps.
|
|
|
|
'''
|
|
return {
|
|
'basis': 'sorted_unique_valid_timestamps',
|
|
'expected_period_s': period_s,
|
|
'verifiable': verifiable,
|
|
'count': 0,
|
|
'aligned_count': 0,
|
|
'misaligned_count': 0,
|
|
'subperiod_count': 0,
|
|
'missing_samples_total': 0,
|
|
'details_count': 0,
|
|
'details_truncated': False,
|
|
'intervals': [],
|
|
'subperiod_intervals': [],
|
|
}
|
|
|
|
|
|
def _audit_timestamps(
|
|
df: pl.DataFrame,
|
|
period_s: int,
|
|
max_gaps: int,
|
|
|
|
) -> tuple[dict, dict]:
|
|
'''
|
|
Audit physical timestamp ordering and chronological gaps.
|
|
|
|
'''
|
|
if 'time' not in df.columns:
|
|
return (
|
|
{
|
|
'present': False,
|
|
'numeric': False,
|
|
'strictly_increasing': False,
|
|
},
|
|
_empty_gaps(period_s),
|
|
)
|
|
|
|
series: pl.Series = df['time']
|
|
counts: dict[str, int|None] = _numeric_counts(series)
|
|
if not series.dtype.is_numeric():
|
|
return (
|
|
{
|
|
'present': True,
|
|
'numeric': False,
|
|
**counts,
|
|
'strictly_increasing': False,
|
|
},
|
|
_empty_gaps(period_s),
|
|
)
|
|
|
|
values, finite = _numeric_values(series)
|
|
valid: np.ndarray = values[finite]
|
|
unique: np.ndarray = np.unique(valid)
|
|
adjacent_valid: np.ndarray = finite[:-1] & finite[1:]
|
|
deltas: np.ndarray = np.diff(values)
|
|
physical_deltas: np.ndarray = deltas[adjacent_valid]
|
|
|
|
zero_delta_count: int = int(np.count_nonzero(
|
|
physical_deltas == 0
|
|
))
|
|
negative_delta_count: int = int(np.count_nonzero(
|
|
physical_deltas < 0
|
|
))
|
|
first: float|int|None = None
|
|
last: float|int|None = None
|
|
if values.size:
|
|
if finite[0]:
|
|
first = _number(values[0])
|
|
if finite[-1]:
|
|
last = _number(values[-1])
|
|
|
|
minimum: float|int|None = None
|
|
maximum: float|int|None = None
|
|
if valid.size:
|
|
minimum = _number(np.min(valid))
|
|
maximum = _number(np.max(valid))
|
|
|
|
chronological: np.ndarray = unique[unique > 0]
|
|
gap_deltas: np.ndarray = np.diff(chronological)
|
|
gap_indexes: np.ndarray = np.flatnonzero(
|
|
gap_deltas > period_s
|
|
)
|
|
gap_values: np.ndarray = gap_deltas[gap_indexes]
|
|
integer_gaps: bool = np.issubdtype(
|
|
gap_values.dtype,
|
|
np.integer,
|
|
)
|
|
if integer_gaps:
|
|
aligned: np.ndarray = gap_values % period_s == 0
|
|
else:
|
|
aligned = np.isclose(
|
|
gap_values % period_s,
|
|
0,
|
|
)
|
|
aligned_count: int = int(np.count_nonzero(aligned))
|
|
if integer_gaps:
|
|
missing_total: int = sum(
|
|
int(delta) // period_s - 1
|
|
for delta in gap_values[aligned]
|
|
)
|
|
else:
|
|
missing_total = sum(
|
|
int(round(float(delta) / period_s)) - 1
|
|
for delta in gap_values[aligned]
|
|
)
|
|
subperiod_indexes: np.ndarray = np.flatnonzero(
|
|
gap_deltas < period_s
|
|
)
|
|
intervals: list[dict] = []
|
|
for gap_index in gap_indexes[:max_gaps]:
|
|
left: float = chronological[gap_index]
|
|
right: float = chronological[gap_index + 1]
|
|
delta: float = right - left
|
|
period_multiple: bool = bool(np.isclose(
|
|
delta % period_s,
|
|
0,
|
|
))
|
|
missing_samples: int|None = None
|
|
if period_multiple:
|
|
missing_samples = (
|
|
int(delta) // period_s - 1
|
|
if integer_gaps
|
|
else int(round(float(delta) / period_s)) - 1
|
|
)
|
|
|
|
left_value: float|int = _number(left)
|
|
right_value: float|int = _number(right)
|
|
intervals.append({
|
|
'left_timestamp': left_value,
|
|
'right_timestamp': right_value,
|
|
'left_utc': _utc_str(left_value),
|
|
'right_utc': _utc_str(right_value),
|
|
'delta_s': _number(delta),
|
|
'period_multiple': period_multiple,
|
|
'missing_samples': missing_samples,
|
|
'classification': 'unclassified',
|
|
})
|
|
|
|
subperiod_intervals: list[dict] = []
|
|
remaining_details: int = max(0, max_gaps - len(intervals))
|
|
for step_index in subperiod_indexes[:remaining_details]:
|
|
left = _number(chronological[step_index])
|
|
right = _number(chronological[step_index + 1])
|
|
subperiod_intervals.append({
|
|
'left_timestamp': left,
|
|
'right_timestamp': right,
|
|
'left_utc': _utc_str(left),
|
|
'right_utc': _utc_str(right),
|
|
'delta_s': _number(right - left),
|
|
})
|
|
|
|
gap_count: int = gap_indexes.size
|
|
gaps: dict = {
|
|
'basis': 'sorted_unique_valid_timestamps',
|
|
'expected_period_s': period_s,
|
|
'verifiable': chronological.size >= 2,
|
|
'count': gap_count,
|
|
'aligned_count': aligned_count,
|
|
'misaligned_count': gap_count - aligned_count,
|
|
'missing_samples_total': missing_total,
|
|
'subperiod_count': subperiod_indexes.size,
|
|
'details_count': (
|
|
len(intervals)
|
|
+ len(subperiod_intervals)
|
|
),
|
|
'details_truncated': bool(
|
|
gap_count > len(intervals)
|
|
or
|
|
subperiod_indexes.size > len(subperiod_intervals)
|
|
),
|
|
'intervals': intervals,
|
|
'subperiod_intervals': subperiod_intervals,
|
|
}
|
|
timestamps: dict = {
|
|
'present': True,
|
|
'numeric': True,
|
|
'first_in_file': first,
|
|
'last_in_file': last,
|
|
'minimum': minimum,
|
|
'maximum': maximum,
|
|
'minimum_utc': _utc_str(minimum),
|
|
'maximum_utc': _utc_str(maximum),
|
|
**counts,
|
|
'zero': int(np.count_nonzero(valid == 0)),
|
|
'negative': int(np.count_nonzero(valid < 0)),
|
|
'fractional': int(np.count_nonzero(
|
|
valid != np.floor(valid)
|
|
)),
|
|
'unique': unique.size,
|
|
'duplicate_excess': valid.size - unique.size,
|
|
'adjacent_zero_delta': zero_delta_count,
|
|
'negative_delta': negative_delta_count,
|
|
'non_positive_delta': (
|
|
zero_delta_count
|
|
+ negative_delta_count
|
|
),
|
|
'strictly_increasing': bool(
|
|
values.size > 0
|
|
and
|
|
values.size == valid.size
|
|
and
|
|
np.all(np.diff(values) > 0)
|
|
),
|
|
}
|
|
return timestamps, gaps
|
|
|
|
|
|
def _audit_index(df: pl.DataFrame) -> dict:
|
|
'''
|
|
Verify persisted indexes match physical row positions.
|
|
|
|
'''
|
|
if 'index' not in df.columns:
|
|
return {
|
|
'present': False,
|
|
'numeric': False,
|
|
'canonical_from_zero': False,
|
|
}
|
|
|
|
series: pl.Series = df['index']
|
|
counts: dict[str, int|None] = _numeric_counts(series)
|
|
if not series.dtype.is_numeric():
|
|
return {
|
|
'present': True,
|
|
'numeric': False,
|
|
'dtype': str(series.dtype),
|
|
**counts,
|
|
'canonical_from_zero': False,
|
|
}
|
|
|
|
values, finite = _numeric_values(series)
|
|
expected: np.ndarray = np.arange(df.height)
|
|
mismatch_count: int = int(np.count_nonzero(
|
|
~finite
|
|
|
|
|
(values != expected)
|
|
))
|
|
valid: np.ndarray = values[finite]
|
|
unique_count: int = np.unique(valid).size
|
|
return {
|
|
'present': True,
|
|
'numeric': True,
|
|
'dtype': str(series.dtype),
|
|
'first': (
|
|
_number(values[0])
|
|
if values.size and finite[0]
|
|
else None
|
|
),
|
|
'last': (
|
|
_number(values[-1])
|
|
if values.size and finite[-1]
|
|
else None
|
|
),
|
|
**counts,
|
|
'duplicate_excess': valid.size - unique_count,
|
|
'non_unit_step': int(np.count_nonzero(
|
|
np.diff(valid) != 1
|
|
)),
|
|
'row_position_mismatch': mismatch_count,
|
|
'contiguous': bool(
|
|
values.size > 0
|
|
and
|
|
valid.size == values.size
|
|
and
|
|
np.all(np.diff(values) == 1)
|
|
),
|
|
'canonical_from_zero': bool(
|
|
values.size > 0
|
|
and
|
|
values.size == expected.size
|
|
and
|
|
mismatch_count == 0
|
|
),
|
|
}
|
|
|
|
|
|
def audit_ohlcv_frame(
|
|
df: pl.DataFrame,
|
|
fqme: str,
|
|
period_s: int,
|
|
max_gaps: int = 100,
|
|
|
|
) -> dict:
|
|
'''
|
|
Inspect a raw persisted frame without normalizing evidence.
|
|
|
|
'''
|
|
if period_s < 1:
|
|
raise ValueError('Audit period must be positive')
|
|
if max_gaps < 0:
|
|
raise ValueError('Maximum gap details must be non-negative')
|
|
|
|
expected_columns: list[str] = [
|
|
name
|
|
for name, _ in def_iohlcv_fields
|
|
]
|
|
expected_dtypes: dict[str, str] = {
|
|
name: str(pl.Int64 if field_type is int else pl.Float64)
|
|
for name, field_type in def_iohlcv_fields
|
|
}
|
|
actual_dtypes: dict[str, str] = {
|
|
name: str(dtype)
|
|
for name, dtype in df.schema.items()
|
|
}
|
|
missing: list[str] = sorted(
|
|
set(expected_columns).difference(df.columns)
|
|
)
|
|
extra: list[str] = sorted(
|
|
set(df.columns).difference(expected_columns)
|
|
)
|
|
column_order_ok: bool = df.columns == expected_columns
|
|
dtypes_ok: bool = all(
|
|
actual_dtypes.get(name) == dtype
|
|
for name, dtype in expected_dtypes.items()
|
|
)
|
|
|
|
timestamps, gaps = _audit_timestamps(
|
|
df,
|
|
period_s,
|
|
max_gaps,
|
|
)
|
|
index: dict = _audit_index(df)
|
|
|
|
nulls: dict[str, int|None] = {}
|
|
nans: dict[str, int|None] = {}
|
|
infinities: dict[str, int|None] = {}
|
|
all_values_finite: bool = True
|
|
for field in _value_fields:
|
|
if field not in df.columns:
|
|
nulls[field] = None
|
|
nans[field] = None
|
|
infinities[field] = None
|
|
all_values_finite = False
|
|
continue
|
|
|
|
counts = _numeric_counts(df[field])
|
|
nulls[field] = counts['null']
|
|
nans[field] = counts['nan']
|
|
pos_inf: int|None = counts['positive_infinity']
|
|
neg_inf: int|None = counts['negative_infinity']
|
|
infinities[field] = (
|
|
None
|
|
if pos_inf is None or neg_inf is None
|
|
else pos_inf + neg_inf
|
|
)
|
|
if counts['nonfinite'] != 0:
|
|
all_values_finite = False
|
|
|
|
schema_ok: bool = bool(
|
|
not missing
|
|
and
|
|
not extra
|
|
and
|
|
column_order_ok
|
|
and
|
|
dtypes_ok
|
|
)
|
|
violations: list[str] = []
|
|
if df.is_empty():
|
|
violations.append('empty_frame')
|
|
if missing:
|
|
violations.append('missing_columns')
|
|
if extra:
|
|
violations.append('extra_columns')
|
|
if not column_order_ok:
|
|
violations.append('column_order')
|
|
if not dtypes_ok:
|
|
violations.append('canonical_dtypes')
|
|
if not all_values_finite:
|
|
violations.append('nonfinite_ohlcv')
|
|
if timestamps.get('nonfinite') != 0:
|
|
violations.append('nonfinite_timestamps')
|
|
if (
|
|
timestamps.get('zero', 0)
|
|
or
|
|
timestamps.get('negative', 0)
|
|
):
|
|
violations.append('nonpositive_timestamps')
|
|
if timestamps.get('fractional', 0):
|
|
violations.append('fractional_timestamps')
|
|
if timestamps.get('duplicate_excess', 0):
|
|
violations.append('duplicate_timestamps')
|
|
if not timestamps.get('strictly_increasing', False):
|
|
violations.append('timestamp_order')
|
|
if not index.get('canonical_from_zero', False):
|
|
violations.append('index_not_canonical')
|
|
if gaps.get('misaligned_count', 0):
|
|
violations.append('misaligned_time_gap')
|
|
if gaps.get('subperiod_count', 0):
|
|
violations.append('subperiod_time_step')
|
|
|
|
warnings: list[str] = []
|
|
if gaps['count']:
|
|
warnings.append('positive_time_gaps_unclassified')
|
|
|
|
structural_ok: bool = not violations
|
|
gap_free: bool|None = (
|
|
gaps['count'] == 0
|
|
if gaps['verifiable']
|
|
else None
|
|
)
|
|
return {
|
|
'audit_schema': AUDIT_SCHEMA,
|
|
'fqme': fqme,
|
|
'period_s': period_s,
|
|
'schema': {
|
|
'columns': df.columns,
|
|
'dtypes': actual_dtypes,
|
|
'expected_columns': expected_columns,
|
|
'expected_dtypes': expected_dtypes,
|
|
'missing_columns': missing,
|
|
'extra_columns': extra,
|
|
'column_order_ok': column_order_ok,
|
|
'canonical_dtypes_ok': dtypes_ok,
|
|
'canonical': schema_ok,
|
|
},
|
|
'rows': {
|
|
'count': df.height,
|
|
'empty': df.is_empty(),
|
|
},
|
|
'timestamps': timestamps,
|
|
'index': index,
|
|
'values': {
|
|
'null_by_column': nulls,
|
|
'nan_by_column': nans,
|
|
'infinity_by_column': infinities,
|
|
'all_finite': all_values_finite,
|
|
},
|
|
'gaps': gaps,
|
|
'result': {
|
|
'structural_ok': structural_ok,
|
|
'gap_free': gap_free,
|
|
'qualification_ok': bool(structural_ok and gap_free),
|
|
'violations': violations,
|
|
'warnings': warnings,
|
|
},
|
|
}
|
|
|
|
|
|
def audit_ohlcv_parquet(
|
|
path: Path,
|
|
fqme: str,
|
|
period_s: int,
|
|
max_gaps: int = 100,
|
|
snapshot: Path|None = None,
|
|
|
|
) -> dict:
|
|
'''
|
|
Read and audit one Parquet file without touching storage state.
|
|
|
|
'''
|
|
path = path.expanduser().absolute()
|
|
descriptor: int = os.open(
|
|
path,
|
|
os.O_RDONLY
|
|
| os.O_CLOEXEC
|
|
| os.O_NOFOLLOW
|
|
| os.O_NONBLOCK,
|
|
)
|
|
stat = os.fstat(descriptor)
|
|
if not S_ISREG(stat.st_mode):
|
|
os.close(descriptor)
|
|
raise ValueError('Audit source must be a regular file')
|
|
with os.fdopen(descriptor, 'rb') as file:
|
|
checksum: str = _sha256(file)
|
|
if snapshot is not None:
|
|
snapshot_is_symlink: bool = snapshot.is_symlink()
|
|
snapshot = snapshot.resolve()
|
|
if snapshot == path:
|
|
raise ValueError(
|
|
'Snapshot path can not replace source'
|
|
)
|
|
if snapshot_is_symlink:
|
|
raise ValueError(
|
|
'Snapshot path can not be a symlink'
|
|
)
|
|
file.seek(0)
|
|
created: bool = False
|
|
try:
|
|
with snapshot.open('xb') as snapshot_file:
|
|
created = True
|
|
os.fchmod(snapshot_file.fileno(), 0o444)
|
|
shutil.copyfileobj(file, snapshot_file)
|
|
except BaseException:
|
|
if created:
|
|
snapshot.unlink(missing_ok=True)
|
|
raise
|
|
file.seek(0)
|
|
frame: pl.DataFrame = pl.read_parquet(file)
|
|
report: dict = audit_ohlcv_frame(
|
|
frame,
|
|
fqme,
|
|
period_s,
|
|
max_gaps,
|
|
)
|
|
generated_at: float = datetime.now(tz=UTC).timestamp()
|
|
report['generated_at_utc'] = _utc_str(generated_at)
|
|
report['source'] = {
|
|
'path': str(path),
|
|
'size_bytes': stat.st_size,
|
|
'mtime_ns': stat.st_mtime_ns,
|
|
'sha256': checksum,
|
|
}
|
|
return report
|