m/market labResearch, with a record.
Source on GitHub ↗

verify_real_sample.py

1"""Independently check an ITCH replay CSV against the original BinaryFILE feed.23Usage: python3 tests/verify_real_sample.py INPUT.bin OUTPUT.csv --symbol AAPL --depth 54The script checks every CSV row's event identity and rebuilds book depth at5deterministically selected rows from its own live-order table.6"""78import argparse9import csv10import mmap11import random12import struct13from collections import Counter14from pathlib import Path151617HEADER = [18    "frame", "timestamp_ns", "type", "bid", "bid_shares", "ask",19    "ask_shares", "bid_levels", "ask_levels", "bid_depth", "ask_depth",20]21ORDER_TYPES = b"AFECXDU"222324def number(feed, offset, size):25    return int.from_bytes(feed[offset:offset + size], "big")262728def price(raw):29    return f"{raw // 10000}.{raw % 10000:04d}"303132def expected_book(orders, depth):33    totals = {ord("B"): {}, ord("S"): {}}34    for side, shares, raw_price in orders.values():35        levels = totals[side]36        levels[raw_price] = levels.get(raw_price, 0) + shares37    bids = sorted(totals[ord("B")].items(), reverse=True)38    asks = sorted(totals[ord("S")].items())3940    def top(levels):41        return (price(levels[0][0]), str(levels[0][1])) if levels else ("", "")4243    def levels_text(levels):44        return "|".join(f"{price(p)}:{shares}" for p, shares in levels[:depth])4546    bid, bid_shares = top(bids)47    ask, ask_shares = top(asks)48    return [bid, bid_shares, ask, ask_shares, str(len(bids)), str(len(asks)),49            levels_text(bids), levels_text(asks)]505152def sample_positions(row_count, count, seed):53    edge = min(25, row_count)54    selected = set(range(1, edge + 1))55    selected.update(range(max(1, row_count - edge + 1), row_count + 1))56    if row_count:57        selected.update(1 + (i * (row_count - 1)) // max(1, count - 1)58                        for i in range(count))59        selected.update(random.Random(seed).sample(range(1, row_count + 1),60                                                  min(count, row_count)))61    return selected626364def check(input_path, csv_path, symbol, depth, samples, seed):65    with csv_path.open(newline="") as source:66        reader = csv.reader(source)67        if next(reader, None) != HEADER:68            raise ValueError("CSV header differs from the expected format")69        row_count = sum(1 for _ in reader)70    selected = sample_positions(row_count, samples, seed)71    expected_symbol = symbol.encode("ascii").ljust(8, b" ")72    orders = {}73    types = Counter()74    sample_types = Counter()75    target_locate = None76    rows_checked = 077    samples_checked = 078    frame = 07980    with input_path.open("rb") as binary, csv_path.open(newline="") as source:81        feed = mmap.mmap(binary.fileno(), 0, access=mmap.ACCESS_READ)82        reader = csv.reader(source)83        next(reader)84        offset = 085        end = len(feed)86        while offset < end:87            if offset + 2 > end:88                raise ValueError(f"truncated length after frame {frame}")89            length = struct.unpack_from(">H", feed, offset)[0]90            offset += 291            if length == 0:92                if offset != end:93                    raise ValueError("bytes follow the BinaryFILE end marker")94                break95            frame += 196            if offset + length > end:97                raise ValueError(f"truncated payload at frame {frame}")98            kind = feed[offset]99            locate = struct.unpack_from(">H", feed, offset + 1)[0]100            if kind == ord("R") and feed[offset + 11:offset + 19] == expected_symbol:101                if target_locate is not None and target_locate != locate:102                    raise ValueError(f"AAPL locate changed at frame {frame}")103                target_locate = locate104            elif kind in ORDER_TYPES and locate == target_locate:105                if kind in (ord("A"), ord("F")):106                    if feed[offset + 24:offset + 32] != expected_symbol:107                        raise ValueError(f"symbol/locate mismatch at frame {frame}")108                    order_id = number(feed, offset + 11, 8)109                    if order_id in orders:110                        raise ValueError(f"duplicate order at frame {frame}")111                    orders[order_id] = (feed[offset + 19],112                                        number(feed, offset + 20, 4),113                                        number(feed, offset + 32, 4))114                elif kind == ord("U"):115                    old_id = number(feed, offset + 11, 8)116                    new_id = number(feed, offset + 19, 8)117                    side, _, _ = orders.pop(old_id)118                    if new_id in orders:119                        raise ValueError(f"duplicate replacement at frame {frame}")120                    orders[new_id] = (side, number(feed, offset + 27, 4),121                                      number(feed, offset + 31, 4))122                else:123                    order_id = number(feed, offset + 11, 8)124                    side, shares, raw_price = orders[order_id]125                    if kind == ord("D"):126                        del orders[order_id]127                    else:128                        removed = number(feed, offset + 19, 4)129                        if not 0 < removed <= shares:130                            raise ValueError(f"invalid reduction at frame {frame}")131                        if removed == shares:132                            del orders[order_id]133                        else:134                            orders[order_id] = (side, shares - removed, raw_price)135136                rows_checked += 1137                row = next(reader, None)138                if row is None or len(row) != len(HEADER):139                    raise ValueError(f"missing/malformed CSV row {rows_checked}")140                identity = [str(frame), str(number(feed, offset + 5, 6)), chr(kind)]141                if row[:3] != identity:142                    raise ValueError(f"CSV event mismatch at row {rows_checked}: "143                                     f"{row[:3]} != {identity}")144                types[chr(kind)] += 1145                # Rare event types should be represented even when the random146                # and evenly spaced positions do not happen to hit them.147                if rows_checked in selected or sample_types[chr(kind)] < 5:148                    expected = expected_book(orders, depth)149                    if row[3:] != expected:150                        mismatches = [(HEADER[i + 3], row[i + 3], value)151                                      for i, value in enumerate(expected)152                                      if row[i + 3] != value]153                        raise ValueError(f"book mismatch at row {rows_checked}, frame "154                                         f"{frame}: {mismatches}")155                    samples_checked += 1156                    sample_types[chr(kind)] += 1157            offset += length158        if next(reader, None) is not None or rows_checked != row_count:159            raise ValueError("CSV has extra rows or row count changed")160        feed.close()161162    print(f"PASS: {frame:,} frames; {rows_checked:,} CSV identities checked; "163          f"{samples_checked:,} full-book samples matched; "164          f"{len(orders):,} live orders at end")165    print("Event counts:", dict(sorted(types.items())))166    print("Sample event counts:", dict(sorted(sample_types.items())))167    print(f"Sampling: first/last 25 rows, first 5 of each event type, "168          f"{samples} evenly spread rows, {samples} random rows (seed {seed})")169170171if __name__ == "__main__":172    parser = argparse.ArgumentParser(description=__doc__)173    parser.add_argument("input", type=Path)174    parser.add_argument("csv", type=Path)175    parser.add_argument("--symbol", default="AAPL")176    parser.add_argument("--depth", type=int, default=5)177    parser.add_argument("--samples", type=int, default=200)178    parser.add_argument("--seed", type=int, default=20260926)179    args = parser.parse_args()180    if args.depth < 1 or args.samples < 1:181        parser.error("--depth and --samples must be positive")182    check(args.input, args.csv, args.symbol, args.depth, args.samples, args.seed)