test_live.py
1"""Actual loopback UDP integration and independent per-event book checks."""2import csv3from concurrent.futures import ThreadPoolExecutor4import io5import random6import socket7import struct8import subprocess9import sys10import tempfile11import time12import unittest13from pathlib import Path14from itch_fixtures import directory, add, reduce, delete, reference_row1516EXE = Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else Path("build/itch_replay").resolve()17ROOT = Path(__file__).resolve().parents[1]18SESSION = b"SIM0000001"192021def wire(seq, messages=(), count=None, session=SESSION):22 return session + struct.pack(">QH", seq, len(messages) if count is None else count) + b"".join(23 struct.pack(">H", len(m)) + m for m in messages)242526def bound():27 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)28 try:29 sock.bind(("127.0.0.1", 0))30 except Exception:31 sock.close()32 raise33 return sock343536def endpoint(sock):37 host, port = sock.getsockname()38 return f"{host}:{port}"394041class LiveTests(unittest.TestCase):42 def setUp(self):43 self.temp = tempfile.TemporaryDirectory()44 self.root = Path(self.temp.name)45 self.source, self.recovery = bound(), bound()46 self.recovery.settimeout(1)47 self.address = None48 self.proc = None49 self.frames = [directory(), add(1, "B", 100, 1000000, timestamp=2),50 add(2, "B", 200, 1000000, timestamp=3),51 reduce("X", 1, 20, timestamp=4), delete(2, timestamp=5), delete(1, timestamp=6)]52 self.messages = [frame[2:] for frame in self.frames]53 self.output = self.root / "live.csv"54 self.output.write_text("OLD")55 self.logfile = self.root / "status.log"5657 def tearDown(self):58 if self.proc and self.proc.poll() is None:59 self.proc.kill()60 self.proc.wait()61 self.source.close()62 self.recovery.close()63 self.temp.cleanup()6465 def start(self, *extra):66 reservation = bound()67 self.address = reservation.getsockname()68 listen = endpoint(reservation)69 reservation.close()70 args = [str(EXE), "--live", "--listen", listen, "--source", endpoint(self.source),71 "--recovery", endpoint(self.recovery), "--session", SESSION.decode(),72 "--symbol", "TEST", "--output", str(self.output), "--check-book",73 "--idle-timeout-ms", "700", "--recovery-timeout-ms", "350", "--retry-ms", "30", *extra]74 with self.logfile.open("w") as log:75 self.proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log)76 deadline = time.monotonic() + 377 while time.monotonic() < deadline:78 if "listening=" in self.logfile.read_text():79 return80 if self.proc.poll() is not None:81 self.fail(self.logfile.read_text())82 time.sleep(0.005)83 self.fail("receiver did not become ready")8485 def send(self, seq, messages=(), count=None, session=SESSION, recovery=False):86 (self.recovery if recovery else self.source).sendto(wire(seq, messages, count, session), self.address)8788 def finish(self, success=True):89 self.proc.communicate(timeout=3)90 log = self.logfile.read_text()91 self.assertEqual(self.proc.returncode, 0 if success else 1, log)92 if not success:93 self.assertEqual(self.output.read_text(), "OLD", log)94 self.assertFalse(list(self.root.glob(".live.csv.partial.*")))95 return log9697 def check_book(self):98 expected = []99 orders = {}100 for seq, timestamp, kind in [(2, 2, "A"), (3, 3, "A"), (4, 4, "X"), (5, 5, "D"), (6, 6, "D")]:101 if seq == 2:102 orders[1] = ("B", 100, 1000000)103 elif seq == 3:104 orders[2] = ("B", 200, 1000000)105 elif seq == 4:106 orders[1] = ("B", 80, 1000000)107 elif seq == 5:108 del orders[2]109 else:110 del orders[1]111 expected.append(reference_row(orders, seq, timestamp, kind, 5))112 self.assertEqual(list(csv.DictReader(io.StringIO(self.output.read_text()))), expected)113 feed = self.root / "fixture.bin"114 feed.write_bytes(b"".join(self.frames) + b"\0\0")115 baseline = subprocess.run([str(EXE), "--input", str(feed), "--symbol", "TEST", "--verify", "--check-book"],116 capture_output=True, check=True)117 self.assertEqual(self.output.read_bytes(), baseline.stdout)118119 def request(self):120 data, client = self.recovery.recvfrom(65536)121 self.assertEqual(data[:10], SESSION)122 self.assertEqual(len(data), 20)123 self.assertEqual(client, self.address)124 return struct.unpack(">QH", data[10:])125126 def test_clean_batched_and_duplicate(self):127 self.start()128 self.send(1, self.messages[:3])129 self.send(2, self.messages[1:4]) # overlap processed plus new message130 self.send(5, self.messages[4:])131 self.send(7, count=65535)132 log = self.finish()133 self.assertIn("duplicates=2", log)134 self.check_book()135136 def test_gap_out_of_order_duplicate_and_recovery(self):137 self.start()138 self.send(1, self.messages[:1])139 self.send(5, self.messages[4:])140 self.send(3, self.messages[2:4])141 self.send(3, self.messages[2:4])142 self.send(7, count=65535)143 seq, count = self.request()144 self.assertEqual(seq, 2)145 self.assertGreaterEqual(count, 1)146 self.send(2, self.messages[1:2], recovery=True)147 log = self.finish()148 self.assertIn("state=RECOVERING", log)149 self.assertIn("state=SYNCED", log)150 self.assertIn("nasdaq_connection=unverified", log)151 self.check_book()152153 def test_lost_first_message_and_partial_recovery(self):154 self.start()155 self.send(6, self.messages[5:])156 self.send(7, count=65535)157 first, count = self.request()158 self.assertEqual((first, count), (1, 5))159 self.send(1, self.messages[:2], recovery=True)160 first, count = self.request()161 self.assertEqual((first, count), (3, 3))162 self.send(3, self.messages[2:5], recovery=True)163 self.finish()164 self.check_book()165166 def test_heartbeat_detects_missing_tail(self):167 self.start()168 self.send(1, self.messages[:4])169 self.send(7) # heartbeat announces sender's next sequence170 self.assertEqual(self.request(), (5, 2))171 self.send(5, self.messages[4:], recovery=True)172 self.send(7, count=65535)173 self.finish()174 self.check_book()175176 def test_end_boundary_recovers_missing_tail(self):177 self.start()178 self.send(1, self.messages[:4])179 self.send(7, count=65535)180 self.assertEqual(self.request(), (5, 2))181 self.send(5, self.messages[4:], recovery=True)182 self.finish()183 self.check_book()184185 def test_missing_end_is_not_success(self):186 self.start()187 self.send(1, self.messages)188 self.assertIn("progress timeout", self.finish(False))189190 def test_silent_socket_timeout(self):191 self.start()192 self.assertIn("progress timeout", self.finish(False))193194 def test_no_recovery_retries_and_fails(self):195 self.start()196 self.send(1, self.messages[:1])197 self.send(3, self.messages[2:])198 self.send(7, count=65535)199 self.assertEqual(self.request(), (2, 1))200 self.assertEqual(self.request(), (2, 1))201 self.assertIn("recovery timeout", self.finish(False))202203 def test_session_mismatch(self):204 self.start()205 self.send(1, self.messages[:1], session=b"OTHER00001")206 self.assertIn("session mismatch", self.finish(False))207208 def test_buffer_message_limit(self):209 self.start("--max-buffer-messages", "2")210 self.send(3, self.messages[2:])211 self.assertIn("buffer limit", self.finish(False))212213 def test_buffer_byte_limit(self):214 self.start("--max-buffer-bytes", "40")215 self.send(3, self.messages[2:4])216 self.assertIn("buffer limit", self.finish(False))217218 def test_conflicting_buffered_duplicate(self):219 self.start()220 self.send(3, self.messages[2:3])221 changed = bytearray(self.messages[2]); changed[-1] ^= 1222 self.send(3, [bytes(changed)])223 self.assertIn("conflicting buffered duplicate", self.finish(False))224225 def test_conflicting_processed_duplicate(self):226 self.start()227 self.send(1, self.messages[:2])228 changed = bytearray(self.messages[1]); changed[-1] ^= 1229 self.send(2, [bytes(changed)])230 self.assertIn("conflicting processed duplicate", self.finish(False))231232 def test_unconfigured_source_ignored(self):233 self.start()234 with bound() as stranger:235 stranger.sendto(b"garbage", self.address)236 self.send(1, self.messages)237 self.send(7, count=65535)238 self.assertIn("ignored_sources=1", self.finish())239 self.check_book()240241 def test_malformed_datagrams(self):242 cases = [b"short", wire(1, self.messages[:1])[:-1], wire(1, count=1),243 wire(1) + b"trailing", wire(1, [b"Z"]), wire(0), wire(2**64-1),244 wire(2**64-2, self.messages[:2]), wire(1, count=65535) + b"x"]245 for data in cases:246 with self.subTest(data=data):247 self.start()248 self.source.sendto(data, self.address)249 self.finish(False)250251 def test_invalid_book_event_fails(self):252 self.start()253 self.send(1, [self.messages[0], delete(999)[2:]])254 self.assertIn("unknown order", self.finish(False))255256 def test_inconsistent_end_boundary(self):257 self.start()258 self.send(3, self.messages[2:3])259 self.send(2, count=65535)260 self.assertIn("inconsistent end", self.finish(False))261262 def test_data_beyond_declared_end(self):263 self.start()264 self.send(4, count=65535) # pending recovery, not complete265 self.send(4, self.messages[3:4])266 self.assertIn("beyond end", self.finish(False))267268 def test_empty_session(self):269 self.start()270 self.send(1, count=65535)271 self.assertIn("frames=0", self.finish())272 self.assertEqual(len(list(csv.DictReader(io.StringIO(self.output.read_text())))), 0)273274 def test_sender_command(self):275 self.start()276 source = endpoint(self.source); recovery = endpoint(self.recovery)277 self.source.close(); self.recovery.close()278 fixture = self.root / "sender.bin"279 sender = subprocess.run([sys.executable, str(ROOT / "tools/network_sender.py"), "--destination",280 f"{self.address[0]}:{self.address[1]}", "--source", source, "--recovery", recovery,281 "--write-fixture", str(fixture), "--serve-seconds", "0.6"], capture_output=True, text=True, timeout=3)282 self.assertEqual(sender.returncode, 0, sender.stderr)283 self.assertIn("request sequence=", sender.stdout)284 self.finish()285 self.check_book()286 self.assertEqual(fixture.read_bytes(), b"".join(self.frames) + b"\0\0")287288 def test_many_events_random_order_and_repeated_recovery(self):289 self.frames = [directory()]290 orders = {}291 expected = []292 for order in range(1, 81):293 seq = len(self.frames) + 1294 side = "B" if order % 2 else "S"295 price = 1000000 + (order % 9) * 100296 shares = 10 + order297 self.frames.append(add(order, side, shares, price, timestamp=seq))298 orders[order] = (side, shares, price)299 expected.append(reference_row(orders, seq, seq, "A", 5))300 for order in range(1, 81):301 seq = len(self.frames) + 1302 self.frames.append(reduce("X", order, 5, timestamp=seq))303 side, shares, price = orders[order]304 orders[order] = (side, shares - 5, price)305 expected.append(reference_row(orders, seq, seq, "X", 5))306 for order in range(80, 0, -1):307 seq = len(self.frames) + 1308 self.frames.append(delete(order, timestamp=seq))309 del orders[order]310 expected.append(reference_row(orders, seq, seq, "D", 5))311 self.messages = [f[2:] for f in self.frames]312 # Two messages per retry can repair the entire prefix inside the313 # recovery budget, even if source delivery is slow or packets are lost.314 self.start("--recovery-timeout-ms", "1000", "--idle-timeout-ms", "1500", "--retry-ms", "5")315 sequence = list(range(1, len(self.messages) + 1))316 random.Random(20260930).shuffle(sequence)317 missing = {1, 31, 70, 111, 190, 241}318319 def send_source():320 for seq in sequence:321 if seq not in missing:322 self.send(seq, self.messages[seq-1:seq])323 if seq % 19 == 0:324 self.send(seq, self.messages[seq-1:seq])325 time.sleep(0.0005)326 self.send(len(self.messages) + 1, count=65535)327328 # Service recovery while source traffic is still being sent. A slow329 # sender must not accidentally withhold every response until timeout.330 with ThreadPoolExecutor(max_workers=1) as sender:331 sent = sender.submit(send_source)332 self.request() # Deliberately drop only the first response.333 deadline = time.monotonic() + 2334 while self.proc.poll() is None and time.monotonic() < deadline:335 try:336 first, count = self.request()337 except socket.timeout:338 continue339 self.send(first, self.messages[first-1:first-1+min(count, 2)], recovery=True)340 sent.result(timeout=3)341 log = self.finish()342 self.assertGreater(log.count("recovery_request "), 1)343 self.assertEqual(list(csv.DictReader(io.StringIO(self.output.read_text()))), expected)344 feed = self.root / "long.bin"345 feed.write_bytes(b"".join(self.frames) + b"\0\0")346 baseline = subprocess.run([str(EXE), "--input", str(feed), "--symbol", "TEST", "--verify"],347 check=True, capture_output=True)348 self.assertEqual(self.output.read_bytes(), baseline.stdout)349350 def test_gap_is_reported_before_any_final_publication(self):351 self.start()352 self.send(1, self.messages[:1])353 self.send(3, self.messages[2:])354 self.request()355 self.assertEqual(self.output.read_text(), "OLD")356 # Status records next=2: no later event has been applied while 2 is missing.357 self.assertIn("state=RECOVERING next_sequence=2", self.logfile.read_text())358 self.send(2, self.messages[1:2], recovery=True)359 self.send(7, count=65535)360 self.finish()361362 def test_network_options_require_live(self):363 result = subprocess.run([str(EXE), "--input", "missing.bin", "--symbol", "TEST", "--retry-ms", "100"],364 capture_output=True, text=True)365 self.assertEqual(result.returncode, 1)366 self.assertIn("network options require --live", result.stderr)367368 def test_cli_rejects_incompatible_and_invalid_options(self):369 for args in [["--input", "x"], ["--verify"], ["--resume", "x"], ["--with-offset"],370 ["--checkpoint-out", "x"], ["--allow-implicit-eof"], ["--retry-ms", "0"],371 ["--session", "short"], ["--listen", "224.0.0.1:1234"],372 ["--listen", "127.0.0.1:99999"], ["--retry-ms", "99999999999999999999"]]:373 with self.subTest(args=args):374 command = [str(EXE), "--live", "--listen", "127.0.0.1:19000", "--source", "127.0.0.1:19001",375 "--recovery", "127.0.0.1:19002", "--session", SESSION.decode(), "--symbol", "TEST", *args]376 result = subprocess.run(command, capture_output=True, timeout=2)377 self.assertEqual(result.returncode, 1, result.stderr)378379380if __name__ == "__main__":381 unittest.main(argv=[sys.argv[0]])