m/market labResearch, with a record.
Source on GitHub ↗

verify_checkpoint_day.py

1"""Full-file checkpoint verification: verified baseline, SIGKILL/restart, exact CSV/core equality.23Writes CSVs, checkpoints, recovery journals, logs and report.json to a NEW directory.4Input is never changed. No Nasdaq data is included in this script.5"""6import argparse7import datetime8import hashlib9import json10import os11from pathlib import Path12import struct13import subprocess14import time151617def file_sha256(path):18    digest = hashlib.sha256()19    with path.open('rb') as source:20        for block in iter(lambda: source.read(1024 * 1024), b''):21            digest.update(block)22    return digest.hexdigest()232425def checkpoint(path):26    data = path.read_bytes()27    body, checksum = data[:-64], data[-64:]28    if hashlib.sha256(body).hexdigest().encode() != checksum:29        raise AssertionError('checkpoint SHA-256 mismatch')30    position = 03132    def take(n):33        nonlocal position34        value = body[position:position+n]35        if len(value) != n:36            raise AssertionError('truncated checkpoint')37        position += n38        return value3940    def number(width=8):41        return int.from_bytes(take(width), 'big')4243    def field():44        return take(number())4546    assert take(8) == b'ITCHCP01'47    version = number(4)48    assert version in (1, 2)49    result = {'input_bytes': number(), 'input_sha256': field().decode(), 'symbol': field().decode()}50    for name in ('next_frame', 'offset', 'selected', 'csv_fnv1a', 'depth'):51        result[name] = number()52    for name in ('with_offset', 'allow_implicit_eof', 'last_end_messages', 'has_locate'):53        result[name] = number(1)54        assert result[name] in (0, 1)55    result['locate'] = number(2)56    result['live_orders'] = number()57    take(result['live_orders'] * 17)58    for name in ('bid_levels', 'ask_levels'):59        result[name] = number()60        take(result[name] * 12)61    # Include every order/level byte, but exclude the run-specific CSV journal binding.62    core = body[:8] + struct.pack('>I', 1) + body[12:position]63    result['core_sha256'] = hashlib.sha256(core).hexdigest()64    if version == 2:65        result['csv_path'] = field().decode()66        result['csv_bytes'] = number()67    assert position == len(body)68    result['frames'] = result['next_frame'] - 169    return result, core707172def exact_csv_equal(left, right):73    rows = 074    digest = hashlib.sha256()75    with left.open('rb') as a, right.open('rb') as b:76        while True:77            x, y = a.read(1024 * 1024), b.read(1024 * 1024)78            if x != y:79                raise AssertionError('full CSV bytes differ')80            if not x:81                break82            rows += x.count(b'\n')83            digest.update(x)84    return rows - 1, digest.hexdigest()858687def main():88    parser = argparse.ArgumentParser(description=__doc__)89    parser.add_argument('input', type=Path)90    parser.add_argument('output_dir', type=Path)91    parser.add_argument('--exe', type=Path, default=Path('build/itch_replay'))92    parser.add_argument('--symbol', default='AAPL')93    parser.add_argument('--depth', type=int, default=5)94    parser.add_argument('--interval', type=int, default=10000000)95    parser.add_argument('--interrupt-at', type=int, nargs='+', default=[20000000, 130000000, 250000000])96    parser.add_argument('--allow-implicit-eof', action='store_true')97    args = parser.parse_args()98    assert args.interval > 099    assert args.interrupt_at == sorted(set(args.interrupt_at)) and args.interrupt_at[0] > 0100    source, root = args.input.resolve(), args.output_dir.resolve()101    root.mkdir(parents=True, exist_ok=False)102    base = [str(args.exe.resolve()), '--input', str(source), '--symbol', args.symbol,103            '--depth', str(args.depth), '--with-offset']104    if args.allow_implicit_eof:105        base.append('--allow-implicit-eof')106    baseline_csv, baseline_cp = root / 'baseline.csv', root / 'baseline.cp'107    recovered_csv, recovered_cp = root / 'recovered.csv', root / 'recovered.cp'108    report = {'status': 'running', 'started_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(),109              'input': str(source), 'interval': args.interval, 'interruptions': [], 'runs': []}110111    def save_report():112        temporary = root / 'report.json.tmp'113        temporary.write_text(json.dumps(report, indent=2) + '\n')114        temporary.replace(root / 'report.json')115116    def run(name, command, target=None):117        print(f'{name}: starting' + (f'; SIGKILL after saved frame >= {target}' if target else ''), flush=True)118        start = time.monotonic()119        with (root / f'{name}.stdout.log').open('wb') as out, (root / f'{name}.stderr.log').open('wb') as err:120            process = subprocess.Popen(command, stdout=out, stderr=err)121            last_stamp, state, next_status = None, None, start + 30122            try:123                while process.poll() is None:124                    if target and recovered_cp.exists():125                        stat = recovered_cp.stat()126                        stamp = (stat.st_mtime_ns, stat.st_size)127                        if stamp != last_stamp:128                            state, _ = checkpoint(recovered_cp)129                            last_stamp = stamp130                        if state and state['frames'] >= target:131                            process.kill()132                            code = process.wait(timeout=10)133                            assert code == -9, f'expected SIGKILL, got {code}'134                            state, _ = checkpoint(recovered_cp)135                            journal = Path(state['csv_path'])136                            assert journal.stat().st_size >= state['csv_bytes']137                            assert not recovered_csv.exists(), 'partial CSV was published'138                            report['interruptions'].append({139                                'requested_frame': target, 'saved_checkpoint': state,140                                'journal_bytes_after_kill': journal.stat().st_size,141                                'returncode': code})142                            print(f'{name}: killed; recoverable frame={state["frames"]}, '143                                  f'CSV rows={state["selected"]}, live orders={state["live_orders"]}', flush=True)144                            break145                    now = time.monotonic()146                    if now >= next_status:147                        suffix = f'; saved frame={state["frames"]}' if state else ''148                        print(f'{name}: running {now-start:.0f}s{suffix}', flush=True)149                        next_status = now + 30150                    time.sleep(0.05)151                else:152                    if target:153                        raise AssertionError(f'{name}: process finished before interruption: {process.returncode}')154                    if process.returncode:155                        raise AssertionError((root / f'{name}.stderr.log').read_text())156                report['runs'].append({'name': name, 'seconds': time.monotonic()-start,157                                      'returncode': process.returncode, 'command': command})158                save_report()159            finally:160                if process.poll() is None:161                    process.kill()162                    process.wait(timeout=10)163        print(f'{name}: finished after {time.monotonic()-start:.1f}s', flush=True)164165    save_report()166    try:167        run('baseline', base + ['--output', str(baseline_csv), '--checkpoint-out', str(baseline_cp), '--verify'])168        baseline, baseline_core = checkpoint(baseline_cp)169        assert args.interrupt_at[-1] < baseline['frames']170        report['baseline'] = baseline171        save_report()172        periodic = ['--output', str(recovered_csv), '--checkpoint-out', str(recovered_cp),173                    '--checkpoint-every-frames', str(args.interval)]174        for index, target in enumerate(args.interrupt_at):175            command = base + periodic176            if index:177                command += ['--resume', str(recovered_cp), '--resume-csv']178            run(f'interruption-{index+1}', command, target)179        run('final-resume', base + periodic + ['--resume', str(recovered_cp), '--resume-csv', '--verify'])180        recovered, recovered_core = checkpoint(recovered_cp)181        assert recovered_core == baseline_core, 'final checkpoint state differs'182        rows, csv_digest = exact_csv_equal(baseline_csv, recovered_csv)183        assert rows == baseline['selected'] == recovered['selected']184        print('Comparing full input SHA-256 with independent hashlib', flush=True)185        input_digest = file_sha256(source)186        assert input_digest == baseline['input_sha256'] == recovered['input_sha256']187        report.update(status='passed', recovered=recovered, csv_rows=rows, csv_sha256=csv_digest,188                      csv_bytes=baseline_csv.stat().st_size, input_sha256=input_digest,189                      csv_exact_equal=True, checkpoint_core_exact_equal=True)190        save_report()191        print(f'PASS: frames={baseline["frames"]}, rows={rows}, '192              f'CSV SHA-256={csv_digest}; final checkpoint core identical', flush=True)193    except BaseException as error:194        report.update(status='failed', error=str(error))195        save_report()196        raise197198199if __name__ == '__main__':200    main()