Source code for sim_platform.research.overfitting

"""Guards against backtest overfitting: deflated Sharpe, PBO and leak-free splits.

A researcher who tries many Agent prompts, rule parameters or model files and keeps the
best one will see a high Sharpe ratio by chance alone. These pure functions measure how
much of a result survives that selection:

- ``probabilistic_sharpe_ratio`` (PSR): the probability that the true Sharpe ratio
  exceeds a benchmark, allowing for sample length, skewness and fat tails.
- ``deflated_sharpe_ratio`` (DSR): PSR against the Sharpe ratio the best of ``n``
  worthless trials would be expected to reach.
- ``probability_of_backtest_overfitting`` (PBO): combinatorially symmetric
  cross-validation over a matrix of candidate returns.
- ``walk_forward_splits`` and ``purged_k_fold_splits``: train/test index ranges that
  never let a test period inform its own training data.

References: Bailey and López de Prado, "The Sharpe Ratio Efficient Frontier" (2012) and
"The Deflated Sharpe Ratio" (2014); Bailey, Borwein, López de Prado and Zhu, "The
Probability of Backtest Overfitting" (2017); López de Prado, "Advances in Financial
Machine Learning" (2018), chapter 7. Sharpe ratios here are per period (not annualised),
because the formulas are stated per observation.
"""

from __future__ import annotations

from dataclasses import dataclass
from itertools import combinations
import math
from statistics import NormalDist
from typing import List, Optional, Sequence, Tuple

EULER_MASCHERONI = 0.5772156649015329
_NORMAL = NormalDist()

# A half-open index range [start, stop) into a return series.
IndexRange = Tuple[int, int]


def _moments(returns: Sequence[float]) -> Tuple[float, float, float, float]:
    """Mean, standard deviation (population), skewness and kurtosis (not excess)."""
    n = len(returns)
    mean = sum(returns) / n
    deviations = [r - mean for r in returns]
    variance = sum(d * d for d in deviations) / n
    if variance <= 0:
        return mean, 0.0, 0.0, 3.0
    sd = math.sqrt(variance)
    skew = sum(d**3 for d in deviations) / n / sd**3
    kurt = sum(d**4 for d in deviations) / n / variance**2
    return mean, sd, skew, kurt


[docs] def per_period_sharpe(returns: Sequence[float]) -> Optional[float]: """Mean over standard deviation of one return series; None when undefined.""" if len(returns) < 2: return None mean, sd, _, _ = _moments(returns) return None if sd == 0 else mean / sd
[docs] def probabilistic_sharpe_ratio( returns: Sequence[float], benchmark_sharpe: float = 0.0 ) -> Optional[float]: """Probability that the true per-period Sharpe ratio exceeds ``benchmark_sharpe``. Args: returns: Per-period returns (for example daily), at least three values. benchmark_sharpe: The per-period Sharpe ratio to beat; 0 tests for any skill. Returns: A probability in [0, 1], or None for fewer than three returns or zero variance. """ if len(returns) < 3: return None mean, sd, skew, kurt = _moments(returns) if sd == 0: return None sharpe = mean / sd denominator = 1 - skew * sharpe + (kurt - 1) / 4 * sharpe**2 if denominator <= 0: return None z = (sharpe - benchmark_sharpe) * math.sqrt(len(returns) - 1) / math.sqrt(denominator) return _NORMAL.cdf(z)
[docs] def expected_maximum_sharpe(trials: int, sharpe_variance: float) -> float: """Sharpe ratio the best of ``trials`` independent zero-skill strategies would reach. Args: trials: Number of configurations tried (prompts, parameters, model files). sharpe_variance: Variance of the per-period Sharpe ratios across those trials. Returns: The expected maximum per-period Sharpe ratio; 0 for a single trial. """ if trials < 1: raise ValueError('trials must be at least 1') if trials == 1 or sharpe_variance <= 0: return 0.0 first = _NORMAL.inv_cdf(1 - 1 / trials) second = _NORMAL.inv_cdf(1 - 1 / (trials * math.e)) return math.sqrt(sharpe_variance) * ((1 - EULER_MASCHERONI) * first + EULER_MASCHERONI * second)
[docs] def deflated_sharpe_ratio( returns: Sequence[float], *, trial_sharpes: Optional[Sequence[float]] = None, trials: Optional[int] = None, sharpe_variance: Optional[float] = None, ) -> Optional[float]: """Probability that the selected strategy's Sharpe ratio is not a selection artefact. Give either the per-period Sharpe ratios of every trial (``trial_sharpes``), or the number of trials with the variance of their Sharpe ratios. A value above 0.95 is the usual threshold for a result that survives the search that produced it. Returns: A probability in [0, 1], or None when ``returns`` cannot define a Sharpe ratio. Raises: ValueError: If neither trial description is given. """ if trial_sharpes is not None: values = list(trial_sharpes) trials = len(values) mean = sum(values) / trials if trials else 0.0 sharpe_variance = sum((v - mean) ** 2 for v in values) / (trials - 1) if trials > 1 else 0.0 if trials is None or sharpe_variance is None: raise ValueError('give trial_sharpes, or trials and sharpe_variance') return probabilistic_sharpe_ratio(returns, expected_maximum_sharpe(trials, sharpe_variance))
[docs] @dataclass(frozen=True) class OverfittingEstimate: """Result of combinatorially symmetric cross-validation. ``probability`` is the share of splits where the in-sample winner ranked in the bottom half out of sample. ``logits`` holds each split's relative out-of-sample rank as log(w / (1 - w)); negative values are the overfit splits. """ probability: float splits: int logits: Tuple[float, ...]
[docs] def probability_of_backtest_overfitting( returns_by_candidate: Sequence[Sequence[float]], blocks: int = 8 ) -> OverfittingEstimate: """Estimate how often choosing the best backtest picks a below-median strategy. Args: returns_by_candidate: One equally long per-period return series for each candidate configuration (at least two candidates). blocks: Even number of contiguous blocks the time axis is cut into; every choice of half the blocks is one in-sample/out-of-sample split. Returns: The estimate over all C(blocks, blocks / 2) splits. Raises: ValueError: For fewer than two candidates, unequal lengths, an odd or too large ``blocks``. """ candidates = [list(r) for r in returns_by_candidate] if len(candidates) < 2: raise ValueError('need at least two candidates') length = len(candidates[0]) if any(len(r) != length for r in candidates): raise ValueError('every candidate needs the same number of returns') if blocks < 2 or blocks % 2 or blocks > length: raise ValueError('blocks must be even and no more than the number of returns') edges = [round(i * length / blocks) for i in range(blocks + 1)] def score(series: List[float], chosen: Sequence[int]) -> float: values = [v for b in chosen for v in series[edges[b] : edges[b + 1]]] sharpe = per_period_sharpe(values) return float('-inf') if sharpe is None else sharpe logits: List[float] = [] count = len(candidates) for train in combinations(range(blocks), blocks // 2): test = [b for b in range(blocks) if b not in train] in_sample = [score(c, train) for c in candidates] best = max(range(count), key=lambda i: in_sample[i]) out_of_sample = [score(c, test) for c in candidates] # Rank 1 is the worst; ties share the lower rank, which is conservative. rank = 1 + sum(1 for v in out_of_sample if v < out_of_sample[best]) relative = rank / (count + 1) logits.append(math.log(relative / (1 - relative))) overfit = sum(1 for value in logits if value <= 0) return OverfittingEstimate(overfit / len(logits), len(logits), tuple(logits))
[docs] def walk_forward_splits( length: int, train: int, test: int, *, step: Optional[int] = None, anchored: bool = False, gap: int = 0, ) -> List[Tuple[IndexRange, IndexRange]]: """Successive train/test windows that always test after they train. Args: length: Number of observations. train: Training window length (the first window's length when ``anchored``). test: Test window length. step: How far each split moves forward; defaults to ``test``. anchored: Keep the training start at 0 so the window grows. gap: Observations skipped between training and test, for labels that look ahead. Returns: ((train_start, train_stop), (test_start, test_stop)) half-open ranges in order. """ if min(train, test) < 1 or gap < 0: raise ValueError('train and test must be positive and gap not negative') step = test if step is None else step if step < 1: raise ValueError('step must be positive') splits = [] start = 0 while start + train + gap + test <= length: train_range = (0 if anchored else start, start + train) test_start = start + train + gap splits.append((train_range, (test_start, test_start + test))) start += step return splits
[docs] def purged_k_fold_splits( length: int, folds: int, *, label_horizon: int = 0, embargo: int = 0 ) -> List[Tuple[List[int], IndexRange]]: """K contiguous test folds with leaking training observations removed. A label computed from the next ``label_horizon`` observations overlaps the test fold when it starts within that distance before the fold, so those observations are purged. ``embargo`` observations after the fold are dropped as well, because serial correlation lets them carry test information backwards. Returns: For each fold, the sorted training indices and the (start, stop) test range. """ if folds < 2 or folds > length: raise ValueError('folds must be between 2 and the number of observations') if label_horizon < 0 or embargo < 0: raise ValueError('label_horizon and embargo must not be negative') edges = [round(i * length / folds) for i in range(folds + 1)] splits = [] for k in range(folds): start, stop = edges[k], edges[k + 1] training = [i for i in range(length) if i + label_horizon < start or i >= stop + embargo] splits.append((training, (start, stop))) return splits