test_replay.py
1"""Small, entirely synthetic BinaryFILE/ITCH 5.0 scenarios (not Nasdaq data)."""2import csv3import hashlib4import io5import random6import struct7import subprocess8import sys9import tempfile10import time11import unittest12from pathlib import Path1314from itch_fixtures import common, frame, directory, add, reduce, delete, replace, reference_row1516EXE = None17SKIPPED_LENGTHS = {18 "S": 12, "H": 25, "Y": 20, "L": 26, "V": 35, "W": 12,19 "K": 28, "J": 35, "h": 21, "P": 44, "Q": 40, "B": 19,20 "I": 50, "N": 20, "O": 48,21}222324252627282930313233343536373839404142class ReplayTests(unittest.TestCase):43 def run_feed(self, data, complete=True, extra_args=()):44 with tempfile.TemporaryDirectory() as directory_name:45 path = Path(directory_name) / "synthetic.bin"46 path.write_bytes(data + (b"\x00\x00" if complete else b""))47 return subprocess.run([str(EXE), "--input", str(path), "--symbol", "TEST", "--verify", *extra_args],48 text=True, capture_output=True)4950 def test_cli_groups_allow_interleaving_and_last_value_wins(self):51 data = directory() + add(1, "B", 10, 1000000) + add(2, "B", 20, 999000)52 expected = self.run_feed(data, extra_args=("--depth", "1", "--with-offset", "--check-book"))53 self.assertEqual(expected.returncode, 0, expected.stderr)54 with tempfile.TemporaryDirectory() as directory_name:55 source = Path(directory_name) / "input.bin"56 output = Path(directory_name) / "output.csv"57 ignored = Path(directory_name) / "ignored.csv"58 source.write_bytes(data + b"\x00\x00")59 result = subprocess.run([60 str(EXE), "--depth", "2", "--input", "missing.bin", "--output", str(ignored),61 "--with-offset", "--symbol", "OTHER", "--check-book", "--input", str(source),62 "--verify", "--output", str(output), "--symbol", "TEST", "--depth", "1",63 ], capture_output=True, text=True)64 self.assertEqual(result.returncode, 0, result.stderr)65 self.assertEqual(output.read_text(), expected.stdout)66 self.assertEqual(result.stdout, "")67 self.assertFalse(ignored.exists())68 self.assertIn("verified=yes", result.stderr)6970 def test_cli_missing_values_and_unknown_options_fail_before_output(self):71 value_options = (72 "--input", "--symbol", "--output", "--depth", "--checkpoint-out", "--resume",73 "--checkpoint-every-frames", "--stop-after-frame", "--listen", "--source",74 "--recovery", "--session", "--idle-timeout-ms", "--recovery-timeout-ms",75 "--retry-ms", "--max-buffer-messages", "--max-buffer-bytes", "--unknown-option",76 )77 with tempfile.TemporaryDirectory() as directory_name:78 source = Path(directory_name) / "input.bin"79 output = Path(directory_name) / "output.csv"80 source.write_bytes(directory() + b"\x00\x00")81 output.write_text("previous result")82 for option in value_options:83 with self.subTest(option=option):84 result = subprocess.run([85 str(EXE), "--input", str(source), "--symbol", "TEST",86 "--output", str(output), option,87 ], capture_output=True, text=True)88 self.assertEqual(result.returncode, 1, result.stderr)89 self.assertIn("usage:", result.stderr)90 self.assertEqual(result.stdout, "")91 self.assertEqual(output.read_text(), "previous result")92 self.assertEqual(sorted(p.name for p in Path(directory_name).iterdir()),93 ["input.bin", "output.csv"])9495 def test_book_lifecycle_and_replay(self):96 data = (directory() + add(1, "B", 100, 1000000) + add(2, "B", 50, 1000000, kind="F") +97 add(3, "S", 80, 1001000) + add(4, "S", 20, 1002000) +98 reduce("E", 1, 40) + reduce("C", 3, 30) + reduce("X", 2, 20) +99 replace(1, 5, 25, 999000) + delete(3) + delete(4))100 result = self.run_feed(data)101 self.assertEqual(result.returncode, 0, result.stderr)102 rows = list(csv.DictReader(io.StringIO(result.stdout)))103 self.assertEqual(len(rows), 10)104 self.assertEqual((rows[2]["bid"], rows[2]["bid_shares"], rows[2]["ask"], rows[2]["ask_shares"]),105 ("100.0000", "150", "100.1000", "80"))106 self.assertEqual((rows[7]["bid"], rows[7]["bid_shares"], rows[7]["bid_levels"]),107 ("100.0000", "30", "2"))108 self.assertEqual((rows[-1]["ask"], rows[-1]["ask_levels"]), ("", "0"))109 self.assertIn("verified=yes", result.stderr)110111 def test_other_symbol_ignored(self):112 system_event = frame(common("S", 0) + b"O")113 result = self.run_feed(system_event + directory() + add(9, "B", 5, 1000000, name="OTHER", locate=8) +114 add(1, "S", 10, 1001000))115 self.assertEqual(result.returncode, 0, result.stderr)116 self.assertEqual(len(list(csv.DictReader(io.StringIO(result.stdout)))), 1)117118 def test_optional_byte_offsets_bound_each_frame(self):119 ignored = frame(common("S", 0) + b"O")120 first = directory()121 second = add(1, "B", 5, 1000000)122 third = add(2, "S", 6, 1001000)123 feed = ignored + first + second + third124125 result = self.run_feed(feed, extra_args=("--with-offset",))126 self.assertEqual(result.returncode, 0, result.stderr)127 rows = list(csv.DictReader(io.StringIO(result.stdout)))128 self.assertEqual(len(rows), 2)129 self.assertEqual(list(rows[0]), ["frame", "timestamp_ns", "type", "bid",130 "bid_shares", "ask", "ask_shares", "bid_levels",131 "ask_levels", "bid_depth", "ask_depth",132 "byte_offset", "byte_end_offset"])133 self.assertEqual([row["byte_offset"] for row in rows],134 [str(len(ignored + first)), str(len(ignored + first + second))])135 self.assertEqual([row["byte_end_offset"] for row in rows],136 [str(len(ignored + first + second)), str(len(feed))])137 for row in rows:138 offset = int(row["byte_offset"])139 length = struct.unpack_from(">H", feed, offset)[0]140 self.assertEqual(int(row["byte_end_offset"]), offset + 2 + length)141 self.assertEqual(chr(feed[offset + 2]), row["type"])142 self.assertIn(length, (36, 40))143144 default = self.run_feed(feed)145 self.assertEqual(default.returncode, 0, default.stderr)146 self.assertNotIn("byte_offset", next(csv.reader(io.StringIO(default.stdout))))147148 def test_every_skipped_type_is_length_checked(self):149 for kind, length in SKIPPED_LENGTHS.items():150 with self.subTest(type=kind):151 valid = frame(kind.encode() + bytes(length - 1))152 result = self.run_feed(directory() + valid + add(1, "B", 5, 1000000))153 self.assertEqual(result.returncode, 0, result.stderr)154 self.assertEqual(len(list(csv.DictReader(io.StringIO(result.stdout)))), 1)155 self.assertIn("frames=3", result.stderr)156157 invalid = frame(kind.encode() + bytes(length - 2))158 result = self.run_feed(directory() + invalid)159 self.assertNotEqual(result.returncode, 0)160 self.assertIn(f"frame 2: invalid {kind} length", result.stderr)161162 def test_every_book_message_type_is_length_checked(self):163 valid_frames = [164 directory(), add(1, "B", 5, 1000000), add(1, "B", 5, 1000000, kind="F"),165 reduce("E", 1, 1), reduce("C", 1, 1), reduce("X", 1, 1),166 delete(1), replace(1, 2, 5, 1000000),167 ]168 for valid in valid_frames:169 kind = chr(valid[2])170 with self.subTest(type=kind):171 shortened = frame(valid[2:-1])172 result = self.run_feed(shortened)173 self.assertNotEqual(result.returncode, 0)174 self.assertIn(f"frame 1: invalid {kind} length", result.stderr)175176 def test_later_corruption_reports_first_bad_frame(self):177 prefix = directory() + add(1, "B", 10, 1000000)178 cases = [179 (prefix + b"\x00", "truncated frame length after frame 2"),180 (prefix + b"\x00\x24A", "frame 3: truncated payload"),181 (prefix + frame(b"A"), "frame 3: invalid A length"),182 (prefix + frame(b"Z"), "frame 3: unknown ITCH type"),183 (prefix, "missing BinaryFILE end marker after frame 2"),184 (prefix + b"\x00\x00X", "bytes after BinaryFILE end marker after frame 2"),185 (prefix + directory(name="TEST", locate=8), "frame 3: symbol changed stock locate"),186 (prefix + directory(name="OTHER", locate=7), "frame 3: stock locate belongs to another symbol"),187 ]188 for data, error in cases:189 with self.subTest(error=error):190 result = self.run_feed(data, complete=False)191 self.assertNotEqual(result.returncode, 0)192 self.assertIn(error, result.stderr)193194 def test_implicit_eof_requires_explicit_option_and_final_end_of_messages(self):195 end_messages = frame(common("S", 0) + b"C")196 prefix = directory() + add(1, "B", 10, 1000000)197 strict = self.run_feed(prefix + end_messages, complete=False)198 self.assertNotEqual(strict.returncode, 0)199 self.assertIn("missing BinaryFILE end marker after frame 3", strict.stderr)200201 allowed = self.run_feed(prefix + end_messages, complete=False,202 extra_args=("--allow-implicit-eof",))203 self.assertEqual(allowed.returncode, 0, allowed.stderr)204 self.assertIn("frames=3", allowed.stderr)205 self.assertEqual(len(list(csv.DictReader(io.StringIO(allowed.stdout)))), 1)206207 for bad in (prefix, prefix + end_messages + add(2, "S", 2, 1001000),208 prefix + end_messages + b"\x00", prefix + frame(common("S", 7) + b"C")):209 with self.subTest(bad=bad[-16:]):210 result = self.run_feed(bad, complete=False,211 extra_args=("--allow-implicit-eof",))212 self.assertNotEqual(result.returncode, 0)213214 def test_output_file_is_published_only_on_success(self):215 with tempfile.TemporaryDirectory() as directory_name:216 root = Path(directory_name)217 source = root / "source.bin"218 destination = root / "book.csv"219 command = [str(EXE), "--input", str(source), "--symbol", "TEST",220 "--output", str(destination), "--verify"]221 valid_prefix = directory() + add(1, "B", 10, 1000000)222223 destination.write_text("previous complete result\n")224 source.write_bytes(valid_prefix + frame(b"Z") + b"\x00\x00")225 failed = subprocess.run(command, text=True, capture_output=True)226 self.assertNotEqual(failed.returncode, 0)227 self.assertIn("frame 3: unknown ITCH type", failed.stderr)228 self.assertEqual(destination.read_text(), "previous complete result\n")229 self.assertEqual(list(root.glob(".book.csv.partial.*")), [])230231 destination.unlink()232 failed_without_old_file = subprocess.run(command, text=True, capture_output=True)233 self.assertNotEqual(failed_without_old_file.returncode, 0)234 self.assertFalse(destination.exists())235 self.assertEqual(list(root.glob(".book.csv.partial.*")), [])236237 source.write_bytes(valid_prefix + b"\x00\x00")238 succeeded = subprocess.run(command, text=True, capture_output=True)239 self.assertEqual(succeeded.returncode, 0, succeeded.stderr)240 rows = list(csv.DictReader(io.StringIO(destination.read_text())))241 self.assertEqual(len(rows), 1)242 self.assertEqual(rows[0]["bid_depth"], "100.0000:10")243 self.assertEqual(list(root.glob(".book.csv.partial.*")), [])244245 published = destination.read_text()246 destination.write_text("older result\n")247 replacement = subprocess.run(command, text=True, capture_output=True)248 self.assertEqual(replacement.returncode, 0, replacement.stderr)249 self.assertEqual(destination.read_text(), published)250251 original_input = source.read_bytes()252 same_file = subprocess.run([str(EXE), "--input", str(source), "--symbol", "TEST",253 "--output", str(source)], text=True, capture_output=True)254 self.assertNotEqual(same_file.returncode, 0)255 self.assertIn("input and output refer to the same file", same_file.stderr)256 self.assertEqual(source.read_bytes(), original_input)257258 def checkpoint_command(self, source, *args):259 return subprocess.run([str(EXE), "--input", str(source), "--symbol", "TEST", *args],260 text=True, capture_output=True)261262 def checkpoint_layout(self, data):263 """Independent parser for v1/v2 metadata and mutation positions."""264 body = data[:-64]265 self.assertEqual(hashlib.sha256(body).hexdigest().encode(), data[-64:])266 self.assertEqual(body[:8], b"ITCHCP01")267 version = struct.unpack_from(">I", body, 8)[0]268 self.assertIn(version, (1, 2))269 pos = 12270 fields = {}271 for key in ("size", "digest", "symbol", "next_frame", "offset", "selected", "hash", "depth"):272 fields[key] = pos273 n = struct.unpack_from(">Q", body, pos)[0]274 pos += 8275 if key in ("digest", "symbol"):276 fields[key + "_value"] = body[pos:pos+n]277 pos += n278 else:279 fields[key + "_value"] = n280 for key in ("with_offset", "implicit_eof", "last_end", "has_locate"):281 fields[key] = pos282 pos += 1283 fields["locate"] = pos284 pos += 2285 fields["order_count"] = pos286 count = struct.unpack_from(">Q", body, pos)[0]287 pos += 8288 fields["orders"] = pos289 pos += count * 17290 fields["bid_count"] = pos291 count = struct.unpack_from(">Q", body, pos)[0]292 fields["bids"] = pos + 8293 pos += 8 + count * 12294 fields["ask_count"] = pos295 fields["asks"] = pos + 8296 pos += 8 + struct.unpack_from(">Q", body, pos)[0] * 12297 fields["version"] = version298 if version == 2:299 fields["csv_binding"] = pos300 count = struct.unpack_from(">Q", body, pos)[0]301 fields["csv_path_value"] = body[pos+8:pos+8+count].decode()302 pos += 8 + count303 fields["csv_bytes"] = pos304 fields["csv_bytes_value"] = struct.unpack_from(">Q", body, pos)[0]305 return fields306307 def test_checkpoint_suffix_and_final_state_at_every_boundary(self):308 payloads = [frame(common("S", 0) + b"O"), directory(),309 add(90, "B", 3, 900000, name="OTHER", locate=8),310 add(1, "B", 100, 1000000), add(2, "B", 50, 1000000, kind="F"),311 add(3, "S", 80, 1001000), add(4, "S", 20, 1002000),312 reduce("E", 1, 40), reduce("C", 3, 30), reduce("X", 2, 50),313 replace(1, 5, 25, 999000), delete(3), delete(4), delete(5),314 add(6, "B", 11, 990000), add(7, "S", 12, 1010000),315 add(8, "B", 13, 990000), frame(common("S", 0) + b"C")]316 with tempfile.TemporaryDirectory() as directory_name:317 root = Path(directory_name)318 source, checkpoint, full_end, resumed_end = [root / n for n in319 ("feed.bin", "saved.cp", "full.cp", "resumed.cp")]320 for flags, marker in (((), b"\0\0"),321 (("--depth", "1", "--with-offset", "--allow-implicit-eof"), b"")):322 source.write_bytes(b"".join(payloads) + marker)323 full = self.checkpoint_command(source, *flags, "--verify", "--check-book",324 "--checkpoint-out", str(full_end))325 self.assertEqual(full.returncode, 0, full.stderr)326 lines = full.stdout.splitlines()327 for boundary in range(len(payloads) + 1):328 with self.subTest(flags=flags, boundary=boundary):329 prefix = self.checkpoint_command(source, *flags, "--verify", "--check-book",330 "--stop-after-frame", str(boundary), "--checkpoint-out", str(checkpoint))331 self.assertEqual(prefix.returncode, 0, prefix.stderr)332 self.assertIn("scope=prefix complete=no", prefix.stderr)333 layout = self.checkpoint_layout(checkpoint.read_bytes())334 self.assertEqual(layout["offset_value"], sum(map(len, payloads[:boundary])))335 self.assertEqual(layout["next_frame_value"], boundary + 1)336 self.assertEqual(layout["digest_value"], hashlib.sha256(source.read_bytes()).hexdigest().encode())337 resumed = self.checkpoint_command(source, *flags, "--verify", "--check-book",338 "--resume", str(checkpoint), "--checkpoint-out", str(resumed_end))339 self.assertEqual(resumed.returncode, 0, resumed.stderr)340 self.assertIn("scope=suffix complete=yes", resumed.stderr)341 expected = [lines[0]] + [line for line in lines[1:] if int(line.split(",")[0]) > boundary]342 self.assertEqual(resumed.stdout.splitlines(), expected)343 self.assertEqual(prefix.stdout.splitlines()[1:] + resumed.stdout.splitlines()[1:], lines[1:])344 # Equal bytes cover every final order, level, locate, counter and cumulative CSV hash.345 self.assertEqual(resumed_end.read_bytes(), full_end.read_bytes())346 terminal = self.checkpoint_command(source, *flags, "--resume", str(full_end))347 self.assertEqual(terminal.returncode, 0, terminal.stderr)348 self.assertEqual(terminal.stdout.splitlines(), [lines[0]])349350 def test_checkpoint_chained_resume_and_empty_session(self):351 with tempfile.TemporaryDirectory() as directory_name:352 root = Path(directory_name)353 source, saved = root / "feed.bin", root / "saved.cp"354 source.write_bytes(directory() + add(1, "B", 10, 1000000) +355 replace(1, 2, 5, 999000) + delete(2) + b"\0\0")356 full = self.checkpoint_command(source)357 first = self.checkpoint_command(source, "--stop-after-frame", "2", "--checkpoint-out", str(saved))358 self.assertEqual(first.returncode, 0, first.stderr)359 middle = self.checkpoint_command(source, "--resume", str(saved), "--stop-after-frame", "3",360 "--checkpoint-out", str(saved), "--verify")361 self.assertEqual(middle.returncode, 0, middle.stderr)362 last = self.checkpoint_command(source, "--resume", str(saved))363 self.assertEqual(last.returncode, 0, last.stderr)364 self.assertEqual(first.stdout.splitlines() + middle.stdout.splitlines()[1:] +365 last.stdout.splitlines()[1:], full.stdout.splitlines())366 source.write_bytes(b"\0\0")367 empty = self.checkpoint_command(source, "--checkpoint-out", str(saved), "--verify")368 self.assertEqual(empty.returncode, 0, empty.stderr)369 restored = self.checkpoint_command(source, "--resume", str(saved), "--verify")370 self.assertEqual(restored.returncode, 0, restored.stderr)371 self.assertEqual(empty.stdout, restored.stdout)372373 def test_periodic_checkpoint_saves_skipped_frames_before_error(self):374 with tempfile.TemporaryDirectory() as directory_name:375 root = Path(directory_name)376 source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"377 good = directory() + frame(common("S", 0) + b"O")378 source.write_bytes(good + frame(b"Z") + b"\0\0")379 output.write_text("previous completed CSV")380 for with_csv in (False, True):381 args = ("--output", str(output)) if with_csv else ()382 result = self.checkpoint_command(source, "--checkpoint-every-frames", "2",383 "--checkpoint-out", str(saved), *args)384 self.assertNotEqual(result.returncode, 0)385 self.assertIn("frame 3: unknown", result.stderr)386 layout = self.checkpoint_layout(saved.read_bytes())387 self.assertEqual(layout["next_frame_value"], 3)388 self.assertEqual(layout["offset_value"], len(good))389 self.assertEqual(layout["selected_value"], 0)390 self.assertEqual(layout["version"], 2 if with_csv else 1)391 if with_csv:392 journal = Path(layout["csv_path_value"])393 self.assertTrue(journal.exists())394 self.assertEqual(journal.stat().st_size, layout["csv_bytes_value"])395 self.assertEqual(output.read_text(), "previous completed CSV")396397 def test_resume_csv_matches_full_replay_and_supports_multiple_stops(self):398 payloads = [directory(), add(1, "B", 10, 1000000),399 frame(common("W", 0) + b"0"), add(2, "S", 20, 1001000),400 replace(1, 3, 5, 999000), reduce("C", 2, 10), delete(3)]401 with tempfile.TemporaryDirectory() as directory_name:402 root = Path(directory_name)403 source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"404 source.write_bytes(b"".join(payloads) + b"\0\0")405 flags = ("--with-offset", "--depth", "1", "--verify", "--check-book")406 full = self.checkpoint_command(source, *flags)407 self.assertEqual(full.returncode, 0, full.stderr)408 for boundary in range(len(payloads) + 1):409 first = self.checkpoint_command(source, *flags, "--stop-after-frame", str(boundary),410 "--checkpoint-every-frames", "2", "--checkpoint-out", str(saved), "--output", str(output))411 self.assertEqual(first.returncode, 0, first.stderr)412 layout = self.checkpoint_layout(saved.read_bytes())413 self.assertEqual(layout["version"], 2)414 # Uncheckpointed bytes, even complete duplicate rows, must be discarded.415 with Path(layout["csv_path_value"]).open("ab") as journal:416 journal.write(b"uncheckpointed partial row,123")417 second = self.checkpoint_command(source, *flags, "--resume", str(saved), "--resume-csv",418 "--output", str(output), "--checkpoint-every-frames", "2")419 self.assertEqual(second.returncode, 0, second.stderr)420 self.assertIn("scope=full complete=yes", second.stderr)421 self.assertEqual(output.read_text(), full.stdout)422 again = self.checkpoint_command(source, *flags, "--resume", str(saved), "--resume-csv",423 "--output", str(output))424 self.assertEqual(again.returncode, 0, again.stderr)425 self.assertEqual(output.read_text(), full.stdout)426 # Chained stops retain the full prefix, not just the newest suffix.427 first = self.checkpoint_command(source, *flags, "--stop-after-frame", "2",428 "--checkpoint-out", str(saved), "--output", str(output))429 self.assertEqual(first.returncode, 0, first.stderr)430 for boundary in (4, 6):431 part = self.checkpoint_command(source, *flags, "--resume", str(saved), "--resume-csv",432 "--output", str(output), "--stop-after-frame", str(boundary))433 self.assertEqual(part.returncode, 0, part.stderr)434 self.assertIn("scope=prefix complete=no", part.stderr)435 end = self.checkpoint_command(source, *flags, "--resume", str(saved), "--resume-csv",436 "--output", str(output))437 self.assertEqual(end.returncode, 0, end.stderr)438 self.assertEqual(output.read_text(), full.stdout)439440 def test_resume_csv_rejects_missing_or_corrupt_prefix_without_overwriting(self):441 with tempfile.TemporaryDirectory() as directory_name:442 root = Path(directory_name)443 source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"444 source.write_bytes(directory() + add(1, "B", 10, 1000000) + delete(1) + b"\0\0")445 first = self.checkpoint_command(source, "--stop-after-frame", "2", "--checkpoint-out", str(saved),446 "--output", str(output))447 self.assertEqual(first.returncode, 0, first.stderr)448 cp = saved.read_bytes()449 layout = self.checkpoint_layout(cp)450 journal = Path(layout["csv_path_value"])451 original = journal.read_bytes()452 for damaged, error in ((original[:-1], "truncated CSV prefix"),453 (b"x" + original[1:], "header mismatch"),454 (original.replace(b"100.0000:10", b"100.0000:11"), "hash mismatch")):455 journal.write_bytes(damaged)456 output.write_text("previous full CSV")457 result = self.checkpoint_command(source, "--resume", str(saved), "--resume-csv", "--output", str(output))458 self.assertNotEqual(result.returncode, 0)459 self.assertIn(error, result.stderr)460 self.assertEqual(saved.read_bytes(), cp)461 self.assertEqual(output.read_text(), "previous full CSV")462 journal.unlink()463 result = self.checkpoint_command(source, "--resume", str(saved), "--resume-csv", "--output", str(output))464 self.assertNotEqual(result.returncode, 0)465 self.assertIn("cannot open saved CSV prefix", result.stderr)466 self.assertEqual(saved.read_bytes(), cp)467 state_only = self.checkpoint_command(source, "--checkpoint-out", str(saved))468 self.assertEqual(state_only.returncode, 0, state_only.stderr)469 result = self.checkpoint_command(source, "--resume", str(saved), "--resume-csv", "--output", str(output))470 self.assertNotEqual(result.returncode, 0)471 self.assertIn("no saved CSV prefix", result.stderr)472473 def test_periodic_checkpoint_recovers_after_process_is_killed(self):474 with tempfile.TemporaryDirectory() as directory_name:475 root = Path(directory_name)476 source, saved, output, expected = [root / name for name in ("feed.bin", "saved.cp", "out.csv", "full.csv")]477 source.write_bytes(directory() + (add(1, "B", 10, 1000000) + delete(1)) * 200000 + b"\0\0")478 output.write_text("previous completed CSV")479 command = [str(EXE), "--input", str(source), "--symbol", "TEST", "--output", str(output),480 "--checkpoint-out", str(saved), "--checkpoint-every-frames", "1000"]481 process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)482 try:483 deadline = time.monotonic() + 15484 while not saved.exists() and process.poll() is None and time.monotonic() < deadline:485 time.sleep(0.005)486 self.assertTrue(saved.exists(), "no periodic checkpoint published")487 self.assertIsNone(process.poll(), "replay completed before interruption")488 process.kill()489 process.communicate(timeout=5)490 finally:491 if process.poll() is None:492 process.kill()493 process.communicate(timeout=5)494 layout = self.checkpoint_layout(saved.read_bytes())495 self.assertLess(layout["next_frame_value"], 400002)496 self.assertEqual(output.read_text(), "previous completed CSV")497 resumed = self.checkpoint_command(source, "--resume", str(saved), "--resume-csv",498 "--output", str(output), "--verify")499 self.assertEqual(resumed.returncode, 0, resumed.stderr)500 full = self.checkpoint_command(source, "--output", str(expected))501 self.assertEqual(full.returncode, 0, full.stderr)502 self.assertEqual(hashlib.sha256(output.read_bytes()).digest(), hashlib.sha256(expected.read_bytes()).digest())503 self.assertEqual(output.stat().st_size, expected.stat().st_size)504505 def test_checkpoint_save_rules_do_not_depend_on_csv_mode(self):506 with tempfile.TemporaryDirectory() as directory_name:507 root = Path(directory_name)508 source, saved, other, output = [root / name for name in509 ("feed.bin", "saved.cp", "other.cp", "out.csv")]510 source.write_bytes(directory() + add(1, "B", 10, 1000000) +511 add(2, "S", 20, 1001000) + delete(1) +512 frame(common("S", 0) + b"O") + delete(2) + b"\0\0")513 full = self.checkpoint_command(source)514 created = self.checkpoint_command(source, "--stop-after-frame", "3", "--checkpoint-out", str(saved),515 "--output", str(output))516 self.assertEqual(created.returncode, 0, created.stderr)517 original = saved.read_bytes()518 suffix = self.checkpoint_command(source, "--resume", str(saved))519 self.assertEqual(suffix.returncode, 0, suffix.stderr)520 self.assertEqual(saved.read_bytes(), original)521 csv_only = self.checkpoint_command(source, "--resume", str(saved), "--resume-csv",522 "--output", str(output), "--verify")523 self.assertEqual(csv_only.returncode, 0, csv_only.stderr)524 self.assertEqual(output.read_text(), full.stdout)525 self.assertEqual(saved.read_bytes(), original, "CSV continuation must not enable saving")526 # Both CSV modes use the same inferred destination when saving is requested.527 modes = ((False, ()), (False, ("--output", str(output))),528 (True, ("--resume-csv", "--output", str(output))))529 for csv_mode, extra in modes:530 saved.write_bytes(original)531 automatic = self.checkpoint_command(source, "--resume", str(saved),532 "--checkpoint-every-frames", "2", "--verify", *extra)533 self.assertEqual(automatic.returncode, 0, automatic.stderr)534 if extra:535 expected = full.stdout.splitlines() if csv_mode else [full.stdout.splitlines()[0]] + [536 row for row in full.stdout.splitlines()[1:] if int(row.split(",")[0]) > 3]537 self.assertEqual(output.read_text().splitlines(), expected)538 self.assertEqual(self.checkpoint_layout(saved.read_bytes())["next_frame_value"], 7)539 saved.write_bytes(original)540 explicit = self.checkpoint_command(source, "--resume", str(saved), "--checkpoint-out", str(other),541 "--checkpoint-every-frames", "2", *extra)542 self.assertEqual(explicit.returncode, 0, explicit.stderr)543 self.assertEqual(saved.read_bytes(), original, "explicit destination must take precedence")544 self.assertEqual(self.checkpoint_layout(other.read_bytes())["next_frame_value"], 7)545 saved.write_bytes(original)546 stopped = self.checkpoint_command(source, "--resume", str(saved), "--stop-after-frame", "4", *extra)547 self.assertEqual(stopped.returncode, 0, stopped.stderr)548 self.assertEqual(self.checkpoint_layout(saved.read_bytes())["next_frame_value"], 5)549550 def test_inferred_periodic_destination_is_written_before_later_failure(self):551 with tempfile.TemporaryDirectory() as directory_name:552 root = Path(directory_name)553 source, saved = root / "feed.bin", root / "saved.cp"554 source.write_bytes(directory() + add(1, "B", 10, 1000000) +555 frame(common("S", 0) + b"O") + delete(1) + frame(b"Z") + b"\0\0")556 first = self.checkpoint_command(source, "--checkpoint-out", str(saved), "--stop-after-frame", "3")557 self.assertEqual(first.returncode, 0, first.stderr)558 resumed = self.checkpoint_command(source, "--resume", str(saved), "--checkpoint-every-frames", "2")559 self.assertNotEqual(resumed.returncode, 0)560 self.assertIn("frame 5: unknown", resumed.stderr)561 state = self.checkpoint_layout(saved.read_bytes())562 self.assertEqual(state["next_frame_value"], 5)563 self.assertEqual(struct.unpack_from(">Q", saved.read_bytes(), state["order_count"])[0], 0)564565 def test_periodic_checkpoint_options_are_explicit_and_validated(self):566 for args, error in ((("--checkpoint-every-frames", "0"), "must be positive"),567 (("--checkpoint-every-frames", "-1"), "must be"),568 (("--checkpoint-every-frames", "abc"), "must be"),569 (("--checkpoint-every-frames", "18446744073709551615"), "must be"),570 (("--checkpoint-every-frames", "10"), "requires --checkpoint-out"),571 (("--resume-csv",), "requires --resume and --output")):572 result = self.run_feed(directory(), extra_args=args)573 self.assertNotEqual(result.returncode, 0)574 self.assertIn(error, result.stderr)575 with tempfile.TemporaryDirectory() as directory_name:576 root = Path(directory_name)577 source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"578 source.write_bytes(directory() + b"\0\0")579 first = self.checkpoint_command(source, "--checkpoint-out", str(saved), "--output", str(output))580 self.assertEqual(first.returncode, 0, first.stderr)581 layout = self.checkpoint_layout(saved.read_bytes())582 journal = Path(layout["csv_path_value"])583 original = journal.read_bytes()584 for args in (("--output", str(journal), "--resume-csv"),585 ("--checkpoint-out", str(journal))):586 result = self.checkpoint_command(source, "--resume", str(saved), *args)587 self.assertNotEqual(result.returncode, 0)588 self.assertIn("must be distinct", result.stderr)589 self.assertEqual(journal.read_bytes(), original)590 result = self.checkpoint_command(source, "--resume", str(saved), "--output", str(output),591 "--checkpoint-out", str(saved), "--checkpoint-every-frames", "1")592 self.assertEqual(result.returncode, 0, result.stderr)593 self.assertEqual(self.checkpoint_layout(saved.read_bytes())["version"], 1)594 self.assertEqual(len(output.read_text().splitlines()), 1)595596 def test_sha256_padding_and_streaming_against_hashlib(self):597 with tempfile.TemporaryDirectory() as directory_name:598 source, saved = Path(directory_name) / "feed.bin", Path(directory_name) / "saved.cp"599 for size in (0, 1, 55, 56, 63, 64, 65, 127, 128, 1000000):600 with self.subTest(size=size):601 source.write_bytes(bytes(i % 251 for i in range(size)))602 result = self.checkpoint_command(source, "--stop-after-frame", "0", "--checkpoint-out", str(saved))603 self.assertEqual(result.returncode, 0, result.stderr)604 layout = self.checkpoint_layout(saved.read_bytes())605 self.assertEqual(layout["size_value"], size)606 self.assertEqual(layout["digest_value"], hashlib.sha256(source.read_bytes()).hexdigest().encode())607608 def test_checkpoint_corruption_and_semantic_validation_preserve_outputs(self):609 with tempfile.TemporaryDirectory() as directory_name:610 root = Path(directory_name)611 source, saved, output, new_cp = [root / n for n in ("feed.bin", "saved.cp", "book.csv", "new.cp")]612 source.write_bytes(directory() + add(1, "B", 10, 1000000) +613 add(2, "B", 20, 1000000) + delete(1) + b"\0\0")614 created = self.checkpoint_command(source, "--stop-after-frame", "3", "--checkpoint-out", str(saved))615 self.assertEqual(created.returncode, 0, created.stderr)616 original = saved.read_bytes()617 f = self.checkpoint_layout(original)618619 def changed(offset, value):620 body = bytearray(original[:-64])621 body[offset:offset+len(value)] = value622 return bytes(body) + hashlib.sha256(body).hexdigest().encode()623624 cases = [625 (original[:-1], "checksum"), (b"broken", "truncated"),626 (original[:30] + bytes([original[30] ^ 1]) + original[31:], "checksum"),627 (changed(0, b"INVALID!"), "magic"),628 (changed(8, struct.pack(">I", 3)), "version"),629 (changed(f["offset"], struct.pack(">Q", f["offset_value"]-1)), "frame boundary"),630 (changed(f["next_frame"], struct.pack(">Q", 3)), "frame/offset"),631 (changed(f["last_end"], b"\1"), "end-message"),632 (changed(f["with_offset"], b"\2"), "invalid flag"),633 (changed(f["orders"] + 8, b"Q"), "invalid live order"),634 (changed(f["orders"] + 9, bytes(4)), "invalid live order"),635 (changed(f["orders"] + 17, original[f["orders"]:f["orders"]+8]), "duplicate order"),636 (changed(f["bids"] + 4, struct.pack(">Q", 999)), "price levels"),637 (changed(f["order_count"], struct.pack(">Q", 2**64-1)), "order count"),638 (changed(f["depth"], struct.pack(">Q", 1001)), "invalid depth"),639 ]640 for damaged, reason in cases:641 with self.subTest(reason=reason):642 saved.write_bytes(damaged)643 output.write_text("previous CSV")644 new_cp.write_bytes(b"previous checkpoint")645 result = self.checkpoint_command(source, "--resume", str(saved),646 "--output", str(output), "--checkpoint-out", str(new_cp), "--verify")647 self.assertNotEqual(result.returncode, 0)648 self.assertIn(reason, result.stderr)649 self.assertEqual(output.read_text(), "previous CSV")650 self.assertEqual(new_cp.read_bytes(), b"previous checkpoint")651 self.assertEqual(list(root.glob(".*.partial.*")), [])652653 saved.write_bytes(original)654 for args, reason in ((("--symbol", "OTHER"), "symbol mismatch"),655 (("--depth", "2"), "options mismatch"),656 (("--with-offset",), "options mismatch"),657 (("--allow-implicit-eof",), "options mismatch")):658 result = self.checkpoint_command(source, "--resume", str(saved), "--output", str(output), *args)659 self.assertNotEqual(result.returncode, 0)660 self.assertIn(reason, result.stderr)661 self.assertEqual(output.read_text(), "previous CSV")662 # Same-size change beyond the checkpoint must still be detected.663 source.write_bytes(source.read_bytes()[:-1] + b"X")664 result = self.checkpoint_command(source, "--resume", str(saved), "--output", str(output))665 self.assertNotEqual(result.returncode, 0)666 self.assertIn("input SHA-256 mismatch", result.stderr)667 self.assertEqual(output.read_text(), "previous CSV")668669 def test_checkpoint_bad_suffix_points_to_untrusted_byte(self):670 with tempfile.TemporaryDirectory() as directory_name:671 root = Path(directory_name)672 source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"673 prefix = directory() + add(1, "B", 10, 1000000)674 source.write_bytes(prefix + frame(b"Z") + b"\0\0")675 created = self.checkpoint_command(source, "--stop-after-frame", "2", "--checkpoint-out", str(saved))676 self.assertEqual(created.returncode, 0, created.stderr)677 original = saved.read_bytes()678 output.write_text("previous")679 result = self.checkpoint_command(source, "--resume", str(saved), "--output", str(output),680 "--checkpoint-out", str(saved))681 self.assertNotEqual(result.returncode, 0)682 self.assertIn("frame 3: unknown ITCH type", result.stderr)683 self.assertIn(f"first untrusted byte_offset={len(prefix)}", result.stderr)684 self.assertEqual(saved.read_bytes(), original)685 self.assertEqual(output.read_text(), "previous")686687 def test_checkpoint_argument_and_path_validation(self):688 for value in ("-1", "1x", "18446744073709551615", "18446744073709551616"):689 result = self.run_feed(directory(), extra_args=("--stop-after-frame", value))690 self.assertNotEqual(result.returncode, 0)691 self.assertIn("--stop-after-frame must be", result.stderr)692 result = self.run_feed(directory(), extra_args=("--stop-after-frame", "0"))693 self.assertNotEqual(result.returncode, 0)694 self.assertIn("requires --checkpoint-out", result.stderr)695 with tempfile.TemporaryDirectory() as directory_name:696 root = Path(directory_name)697 source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"698 source.write_bytes(directory() + b"\0\0")699 saved.write_bytes(b"previous")700 result = self.checkpoint_command(source, "--stop-after-frame", "2", "--checkpoint-out", str(saved))701 self.assertNotEqual(result.returncode, 0)702 self.assertIn("exceeds input", result.stderr)703 self.assertEqual(saved.read_bytes(), b"previous")704 for args in (("--checkpoint-out", str(source)),705 ("--checkpoint-out", str(output), "--output", str(output)),706 ("--resume", str(saved), "--output", str(saved))):707 result = self.checkpoint_command(source, *args)708 self.assertNotEqual(result.returncode, 0)709 self.assertIn("must be distinct", result.stderr)710 alias = root / "alias.cp"711 alias.symlink_to(source)712 result = self.checkpoint_command(source, "--checkpoint-out", str(alias))713 self.assertNotEqual(result.returncode, 0)714 self.assertEqual(source.read_bytes(), directory() + b"\0\0")715716 def test_every_event_matches_independent_reference(self):717 rng = random.Random(20260926)718 payloads = [directory()]719 orders = {}720 expected = {2: [], 5: []}721 next_id = 1722 prices = [990000, 995000, 1000000, 1001000, 1002000, 1010000, 1020000]723724 def record(payload, kind, timestamp):725 payloads.append(payload)726 for limit in expected:727 expected[limit].append(reference_row(orders, len(payloads), timestamp, kind, limit))728729 # Multiple orders share prices; more than five levels exist on both sides.730 for side in ("B", "S"):731 for raw_price in prices:732 for _ in range(2):733 order_id = next_id734 next_id += 1735 shares = rng.randint(1, 500)736 orders[order_id] = (side, shares, raw_price)737 kind = "F" if order_id % 3 == 0 else "A"738 record(add(order_id, side, shares, raw_price, kind=kind,739 timestamp=next_id), kind, next_id)740741 for step in range(250):742 timestamp = 1000 + step743 action = rng.choice(("add", "reduce", "delete", "replace"))744 if not orders:745 action = "add"746 if action == "add":747 order_id = next_id748 next_id += 1749 side = rng.choice(("B", "S"))750 shares = rng.randint(1, 500)751 raw_price = rng.choice(prices)752 orders[order_id] = (side, shares, raw_price)753 kind = rng.choice(("A", "F"))754 record(add(order_id, side, shares, raw_price, kind=kind, timestamp=timestamp),755 kind, timestamp)756 else:757 old_id = rng.choice(tuple(orders))758 side, old_shares, raw_price = orders[old_id]759 if action == "reduce":760 removed = rng.randint(1, old_shares)761 kind = rng.choice(("E", "C", "X"))762 if removed == old_shares:763 del orders[old_id]764 else:765 orders[old_id] = (side, old_shares - removed, raw_price)766 record(reduce(kind, old_id, removed, timestamp=timestamp), kind, timestamp)767 elif action == "delete":768 del orders[old_id]769 record(delete(old_id, timestamp=timestamp), "D", timestamp)770 else:771 new_id = next_id772 next_id += 1773 new_shares = rng.randint(1, 500)774 new_price = rng.choice(prices)775 del orders[old_id]776 orders[new_id] = (side, new_shares, new_price)777 record(replace(old_id, new_id, new_shares, new_price,778 timestamp=timestamp), "U", timestamp)779780 data = b"".join(payloads)781 for limit, rows_expected in expected.items():782 with self.subTest(depth=limit):783 result = self.run_feed(data, extra_args=("--depth", str(limit), "--check-book"))784 self.assertEqual(result.returncode, 0, result.stderr)785 rows_actual = list(csv.DictReader(io.StringIO(result.stdout)))786 self.assertEqual(rows_actual, rows_expected)787 self.assertIn("verified=yes", result.stderr)788 # Restored books must also agree with the independent order model.789 with tempfile.TemporaryDirectory() as directory_name:790 source = Path(directory_name) / "random.bin"791 checkpoint = Path(directory_name) / "random.cp"792 source.write_bytes(data + b"\0\0")793 for boundary in (5, 63, 190, len(payloads)):794 saved = self.checkpoint_command(source, "--depth", str(limit), "--check-book",795 "--stop-after-frame", str(boundary), "--checkpoint-out", str(checkpoint))796 self.assertEqual(saved.returncode, 0, saved.stderr)797 restored = self.checkpoint_command(source, "--depth", str(limit), "--check-book",798 "--resume", str(checkpoint), "--verify")799 self.assertEqual(restored.returncode, 0, restored.stderr)800 self.assertEqual(list(csv.DictReader(io.StringIO(restored.stdout))),801 [row for row in rows_expected if int(row["frame"]) > boundary])802803 def test_empty_and_single_level_depth(self):804 data = directory() + add(1, "B", 5, 1000000) + delete(1)805 result = self.run_feed(data, extra_args=("--depth", "5", "--check-book"))806 self.assertEqual(result.returncode, 0, result.stderr)807 rows = list(csv.DictReader(io.StringIO(result.stdout)))808 self.assertEqual((rows[0]["bid_depth"], rows[0]["ask_depth"]), ("100.0000:5", ""))809 self.assertEqual((rows[1]["bid_depth"], rows[1]["ask_depth"]), ("", ""))810811 def test_depth_argument_validation(self):812 for value in ("0", "-1", "1001", "abc", "2x"):813 with self.subTest(value=value):814 result = self.run_feed(directory(), extra_args=("--depth", value))815 self.assertNotEqual(result.returncode, 0)816 self.assertIn("--depth must be", result.stderr)817818 def test_malformed_and_simulated_missing_or_out_of_order(self):819 scenarios = [820 (b"\x00", "truncated frame length"),821 (b"\x00\x24A", "truncated payload"),822 (frame(b"A"), "invalid A length"),823 (frame(b"Z"), "unknown ITCH type"),824 (directory() + reduce("X", 404, 2), "unknown order reference"), # missing add825 (directory() + reduce("E", 1, 2) + add(1, "B", 10, 1000000), "unknown order reference"), # reordered826 (directory() + add(1, "B", 10, 1000000) + reduce("X", 1, 11), "invalid share reduction"),827 (directory() + add(1, "B", 10, 1000000) + add(1, "B", 10, 1000000), "duplicate order"),828 (directory() + add(1, "B", 0, 1000000), "invalid add fields"),829 (directory() + add(1, "Q", 10, 1000000), "invalid add fields"),830 (directory() + add(1, "B", 10, 0), "invalid add fields"),831 (directory() + add(1, "B", 10, 1000000, name="OTHER"), "add symbol/locate mismatch"),832 (directory() + add(1, "B", 10, 1000000, locate=8), "add symbol/locate mismatch"),833 (directory() + add(1, "B", 10, 1000000) + replace(1, 1, 5, 999000), "invalid replacement"),834 (directory() + add(1, "B", 10, 1000000) + replace(1, 2, 0, 999000), "invalid replacement"),835 ]836 for data, reason in scenarios:837 with self.subTest(reason=reason):838 result = self.run_feed(data, complete=(reason != "truncated frame length"))839 self.assertNotEqual(result.returncode, 0)840 self.assertIn(reason, result.stderr)841 incomplete = self.run_feed(directory(), complete=False)842 self.assertIn("missing BinaryFILE end marker", incomplete.stderr)843 trailing = self.run_feed(directory() + b"\x00\x00X", complete=False)844 self.assertIn("bytes after BinaryFILE end marker", trailing.stderr)845846847if __name__ == "__main__":848 EXE = Path(sys.argv[1]).resolve()849 sys.argv = [sys.argv[0]]850 unittest.main()