verify_checkpoint_day.py
1"""Full-file checkpoint verification: verified baseline, SIGKILL/restart, exact CSV/core equality.23Writes CSVs, checkpoints, recovery journals, logs and report.json to a NEW directory.4Input is never changed. No Nasdaq data is included in this script.5"""6import argparse7import datetime8import hashlib9import json10import os11from pathlib import Path12import struct13import subprocess14import time151617def file_sha256(path):18 digest = hashlib.sha256()19 with path.open('rb') as source:20 for block in iter(lambda: source.read(1024 * 1024), b''):21 digest.update(block)22 return digest.hexdigest()232425def checkpoint(path):26 data = path.read_bytes()27 body, checksum = data[:-64], data[-64:]28 if hashlib.sha256(body).hexdigest().encode() != checksum:29 raise AssertionError('checkpoint SHA-256 mismatch')30 position = 03132 def take(n):33 nonlocal position34 value = body[position:position+n]35 if len(value) != n:36 raise AssertionError('truncated checkpoint')37 position += n38 return value3940 def number(width=8):41 return int.from_bytes(take(width), 'big')4243 def field():44 return take(number())4546 assert take(8) == b'ITCHCP01'47 version = number(4)48 assert version in (1, 2)49 result = {'input_bytes': number(), 'input_sha256': field().decode(), 'symbol': field().decode()}50 for name in ('next_frame', 'offset', 'selected', 'csv_fnv1a', 'depth'):51 result[name] = number()52 for name in ('with_offset', 'allow_implicit_eof', 'last_end_messages', 'has_locate'):53 result[name] = number(1)54 assert result[name] in (0, 1)55 result['locate'] = number(2)56 result['live_orders'] = number()57 take(result['live_orders'] * 17)58 for name in ('bid_levels', 'ask_levels'):59 result[name] = number()60 take(result[name] * 12)61 # Include every order/level byte, but exclude the run-specific CSV journal binding.62 core = body[:8] + struct.pack('>I', 1) + body[12:position]63 result['core_sha256'] = hashlib.sha256(core).hexdigest()64 if version == 2:65 result['csv_path'] = field().decode()66 result['csv_bytes'] = number()67 assert position == len(body)68 result['frames'] = result['next_frame'] - 169 return result, core707172def exact_csv_equal(left, right):73 rows = 074 digest = hashlib.sha256()75 with left.open('rb') as a, right.open('rb') as b:76 while True:77 x, y = a.read(1024 * 1024), b.read(1024 * 1024)78 if x != y:79 raise AssertionError('full CSV bytes differ')80 if not x:81 break82 rows += x.count(b'\n')83 digest.update(x)84 return rows - 1, digest.hexdigest()858687def main():88 parser = argparse.ArgumentParser(description=__doc__)89 parser.add_argument('input', type=Path)90 parser.add_argument('output_dir', type=Path)91 parser.add_argument('--exe', type=Path, default=Path('build/itch_replay'))92 parser.add_argument('--symbol', default='AAPL')93 parser.add_argument('--depth', type=int, default=5)94 parser.add_argument('--interval', type=int, default=10000000)95 parser.add_argument('--interrupt-at', type=int, nargs='+', default=[20000000, 130000000, 250000000])96 parser.add_argument('--allow-implicit-eof', action='store_true')97 args = parser.parse_args()98 assert args.interval > 099 assert args.interrupt_at == sorted(set(args.interrupt_at)) and args.interrupt_at[0] > 0100 source, root = args.input.resolve(), args.output_dir.resolve()101 root.mkdir(parents=True, exist_ok=False)102 base = [str(args.exe.resolve()), '--input', str(source), '--symbol', args.symbol,103 '--depth', str(args.depth), '--with-offset']104 if args.allow_implicit_eof:105 base.append('--allow-implicit-eof')106 baseline_csv, baseline_cp = root / 'baseline.csv', root / 'baseline.cp'107 recovered_csv, recovered_cp = root / 'recovered.csv', root / 'recovered.cp'108 report = {'status': 'running', 'started_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(),109 'input': str(source), 'interval': args.interval, 'interruptions': [], 'runs': []}110111 def save_report():112 temporary = root / 'report.json.tmp'113 temporary.write_text(json.dumps(report, indent=2) + '\n')114 temporary.replace(root / 'report.json')115116 def run(name, command, target=None):117 print(f'{name}: starting' + (f'; SIGKILL after saved frame >= {target}' if target else ''), flush=True)118 start = time.monotonic()119 with (root / f'{name}.stdout.log').open('wb') as out, (root / f'{name}.stderr.log').open('wb') as err:120 process = subprocess.Popen(command, stdout=out, stderr=err)121 last_stamp, state, next_status = None, None, start + 30122 try:123 while process.poll() is None:124 if target and recovered_cp.exists():125 stat = recovered_cp.stat()126 stamp = (stat.st_mtime_ns, stat.st_size)127 if stamp != last_stamp:128 state, _ = checkpoint(recovered_cp)129 last_stamp = stamp130 if state and state['frames'] >= target:131 process.kill()132 code = process.wait(timeout=10)133 assert code == -9, f'expected SIGKILL, got {code}'134 state, _ = checkpoint(recovered_cp)135 journal = Path(state['csv_path'])136 assert journal.stat().st_size >= state['csv_bytes']137 assert not recovered_csv.exists(), 'partial CSV was published'138 report['interruptions'].append({139 'requested_frame': target, 'saved_checkpoint': state,140 'journal_bytes_after_kill': journal.stat().st_size,141 'returncode': code})142 print(f'{name}: killed; recoverable frame={state["frames"]}, '143 f'CSV rows={state["selected"]}, live orders={state["live_orders"]}', flush=True)144 break145 now = time.monotonic()146 if now >= next_status:147 suffix = f'; saved frame={state["frames"]}' if state else ''148 print(f'{name}: running {now-start:.0f}s{suffix}', flush=True)149 next_status = now + 30150 time.sleep(0.05)151 else:152 if target:153 raise AssertionError(f'{name}: process finished before interruption: {process.returncode}')154 if process.returncode:155 raise AssertionError((root / f'{name}.stderr.log').read_text())156 report['runs'].append({'name': name, 'seconds': time.monotonic()-start,157 'returncode': process.returncode, 'command': command})158 save_report()159 finally:160 if process.poll() is None:161 process.kill()162 process.wait(timeout=10)163 print(f'{name}: finished after {time.monotonic()-start:.1f}s', flush=True)164165 save_report()166 try:167 run('baseline', base + ['--output', str(baseline_csv), '--checkpoint-out', str(baseline_cp), '--verify'])168 baseline, baseline_core = checkpoint(baseline_cp)169 assert args.interrupt_at[-1] < baseline['frames']170 report['baseline'] = baseline171 save_report()172 periodic = ['--output', str(recovered_csv), '--checkpoint-out', str(recovered_cp),173 '--checkpoint-every-frames', str(args.interval)]174 for index, target in enumerate(args.interrupt_at):175 command = base + periodic176 if index:177 command += ['--resume', str(recovered_cp), '--resume-csv']178 run(f'interruption-{index+1}', command, target)179 run('final-resume', base + periodic + ['--resume', str(recovered_cp), '--resume-csv', '--verify'])180 recovered, recovered_core = checkpoint(recovered_cp)181 assert recovered_core == baseline_core, 'final checkpoint state differs'182 rows, csv_digest = exact_csv_equal(baseline_csv, recovered_csv)183 assert rows == baseline['selected'] == recovered['selected']184 print('Comparing full input SHA-256 with independent hashlib', flush=True)185 input_digest = file_sha256(source)186 assert input_digest == baseline['input_sha256'] == recovered['input_sha256']187 report.update(status='passed', recovered=recovered, csv_rows=rows, csv_sha256=csv_digest,188 csv_bytes=baseline_csv.stat().st_size, input_sha256=input_digest,189 csv_exact_equal=True, checkpoint_core_exact_equal=True)190 save_report()191 print(f'PASS: frames={baseline["frames"]}, rows={rows}, '192 f'CSV SHA-256={csv_digest}; final checkpoint core identical', flush=True)193 except BaseException as error:194 report.update(status='failed', error=str(error))195 save_report()196 raise197198199if __name__ == '__main__':200 main()