verify_real_sample.py
1"""Independently check an ITCH replay CSV against the original BinaryFILE feed.23Usage: python3 tests/verify_real_sample.py INPUT.bin OUTPUT.csv --symbol AAPL --depth 54The script checks every CSV row's event identity and rebuilds book depth at5deterministically selected rows from its own live-order table.6"""78import argparse9import csv10import mmap11import random12import struct13from collections import Counter14from pathlib import Path151617HEADER = [18 "frame", "timestamp_ns", "type", "bid", "bid_shares", "ask",19 "ask_shares", "bid_levels", "ask_levels", "bid_depth", "ask_depth",20]21ORDER_TYPES = b"AFECXDU"222324def number(feed, offset, size):25 return int.from_bytes(feed[offset:offset + size], "big")262728def price(raw):29 return f"{raw // 10000}.{raw % 10000:04d}"303132def expected_book(orders, depth):33 totals = {ord("B"): {}, ord("S"): {}}34 for side, shares, raw_price in orders.values():35 levels = totals[side]36 levels[raw_price] = levels.get(raw_price, 0) + shares37 bids = sorted(totals[ord("B")].items(), reverse=True)38 asks = sorted(totals[ord("S")].items())3940 def top(levels):41 return (price(levels[0][0]), str(levels[0][1])) if levels else ("", "")4243 def levels_text(levels):44 return "|".join(f"{price(p)}:{shares}" for p, shares in levels[:depth])4546 bid, bid_shares = top(bids)47 ask, ask_shares = top(asks)48 return [bid, bid_shares, ask, ask_shares, str(len(bids)), str(len(asks)),49 levels_text(bids), levels_text(asks)]505152def sample_positions(row_count, count, seed):53 edge = min(25, row_count)54 selected = set(range(1, edge + 1))55 selected.update(range(max(1, row_count - edge + 1), row_count + 1))56 if row_count:57 selected.update(1 + (i * (row_count - 1)) // max(1, count - 1)58 for i in range(count))59 selected.update(random.Random(seed).sample(range(1, row_count + 1),60 min(count, row_count)))61 return selected626364def check(input_path, csv_path, symbol, depth, samples, seed):65 with csv_path.open(newline="") as source:66 reader = csv.reader(source)67 if next(reader, None) != HEADER:68 raise ValueError("CSV header differs from the expected format")69 row_count = sum(1 for _ in reader)70 selected = sample_positions(row_count, samples, seed)71 expected_symbol = symbol.encode("ascii").ljust(8, b" ")72 orders = {}73 types = Counter()74 sample_types = Counter()75 target_locate = None76 rows_checked = 077 samples_checked = 078 frame = 07980 with input_path.open("rb") as binary, csv_path.open(newline="") as source:81 feed = mmap.mmap(binary.fileno(), 0, access=mmap.ACCESS_READ)82 reader = csv.reader(source)83 next(reader)84 offset = 085 end = len(feed)86 while offset < end:87 if offset + 2 > end:88 raise ValueError(f"truncated length after frame {frame}")89 length = struct.unpack_from(">H", feed, offset)[0]90 offset += 291 if length == 0:92 if offset != end:93 raise ValueError("bytes follow the BinaryFILE end marker")94 break95 frame += 196 if offset + length > end:97 raise ValueError(f"truncated payload at frame {frame}")98 kind = feed[offset]99 locate = struct.unpack_from(">H", feed, offset + 1)[0]100 if kind == ord("R") and feed[offset + 11:offset + 19] == expected_symbol:101 if target_locate is not None and target_locate != locate:102 raise ValueError(f"AAPL locate changed at frame {frame}")103 target_locate = locate104 elif kind in ORDER_TYPES and locate == target_locate:105 if kind in (ord("A"), ord("F")):106 if feed[offset + 24:offset + 32] != expected_symbol:107 raise ValueError(f"symbol/locate mismatch at frame {frame}")108 order_id = number(feed, offset + 11, 8)109 if order_id in orders:110 raise ValueError(f"duplicate order at frame {frame}")111 orders[order_id] = (feed[offset + 19],112 number(feed, offset + 20, 4),113 number(feed, offset + 32, 4))114 elif kind == ord("U"):115 old_id = number(feed, offset + 11, 8)116 new_id = number(feed, offset + 19, 8)117 side, _, _ = orders.pop(old_id)118 if new_id in orders:119 raise ValueError(f"duplicate replacement at frame {frame}")120 orders[new_id] = (side, number(feed, offset + 27, 4),121 number(feed, offset + 31, 4))122 else:123 order_id = number(feed, offset + 11, 8)124 side, shares, raw_price = orders[order_id]125 if kind == ord("D"):126 del orders[order_id]127 else:128 removed = number(feed, offset + 19, 4)129 if not 0 < removed <= shares:130 raise ValueError(f"invalid reduction at frame {frame}")131 if removed == shares:132 del orders[order_id]133 else:134 orders[order_id] = (side, shares - removed, raw_price)135136 rows_checked += 1137 row = next(reader, None)138 if row is None or len(row) != len(HEADER):139 raise ValueError(f"missing/malformed CSV row {rows_checked}")140 identity = [str(frame), str(number(feed, offset + 5, 6)), chr(kind)]141 if row[:3] != identity:142 raise ValueError(f"CSV event mismatch at row {rows_checked}: "143 f"{row[:3]} != {identity}")144 types[chr(kind)] += 1145 # Rare event types should be represented even when the random146 # and evenly spaced positions do not happen to hit them.147 if rows_checked in selected or sample_types[chr(kind)] < 5:148 expected = expected_book(orders, depth)149 if row[3:] != expected:150 mismatches = [(HEADER[i + 3], row[i + 3], value)151 for i, value in enumerate(expected)152 if row[i + 3] != value]153 raise ValueError(f"book mismatch at row {rows_checked}, frame "154 f"{frame}: {mismatches}")155 samples_checked += 1156 sample_types[chr(kind)] += 1157 offset += length158 if next(reader, None) is not None or rows_checked != row_count:159 raise ValueError("CSV has extra rows or row count changed")160 feed.close()161162 print(f"PASS: {frame:,} frames; {rows_checked:,} CSV identities checked; "163 f"{samples_checked:,} full-book samples matched; "164 f"{len(orders):,} live orders at end")165 print("Event counts:", dict(sorted(types.items())))166 print("Sample event counts:", dict(sorted(sample_types.items())))167 print(f"Sampling: first/last 25 rows, first 5 of each event type, "168 f"{samples} evenly spread rows, {samples} random rows (seed {seed})")169170171if __name__ == "__main__":172 parser = argparse.ArgumentParser(description=__doc__)173 parser.add_argument("input", type=Path)174 parser.add_argument("csv", type=Path)175 parser.add_argument("--symbol", default="AAPL")176 parser.add_argument("--depth", type=int, default=5)177 parser.add_argument("--samples", type=int, default=200)178 parser.add_argument("--seed", type=int, default=20260926)179 args = parser.parse_args()180 if args.depth < 1 or args.samples < 1:181 parser.error("--depth and --samples must be positive")182 check(args.input, args.csv, args.symbol, args.depth, args.samples, args.seed)