m/market labResearch, with a record.
Source on GitHub ↗

test_live.py

1"""Actual loopback UDP integration and independent per-event book checks."""2import csv3from concurrent.futures import ThreadPoolExecutor4import io5import random6import socket7import struct8import subprocess9import sys10import tempfile11import time12import unittest13from pathlib import Path14from itch_fixtures import directory, add, reduce, delete, reference_row1516EXE = Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else Path("build/itch_replay").resolve()17ROOT = Path(__file__).resolve().parents[1]18SESSION = b"SIM0000001"192021def wire(seq, messages=(), count=None, session=SESSION):22    return session + struct.pack(">QH", seq, len(messages) if count is None else count) + b"".join(23        struct.pack(">H", len(m)) + m for m in messages)242526def bound():27    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)28    try:29        sock.bind(("127.0.0.1", 0))30    except Exception:31        sock.close()32        raise33    return sock343536def endpoint(sock):37    host, port = sock.getsockname()38    return f"{host}:{port}"394041class LiveTests(unittest.TestCase):42    def setUp(self):43        self.temp = tempfile.TemporaryDirectory()44        self.root = Path(self.temp.name)45        self.source, self.recovery = bound(), bound()46        self.recovery.settimeout(1)47        self.address = None48        self.proc = None49        self.frames = [directory(), add(1, "B", 100, 1000000, timestamp=2),50                       add(2, "B", 200, 1000000, timestamp=3),51                       reduce("X", 1, 20, timestamp=4), delete(2, timestamp=5), delete(1, timestamp=6)]52        self.messages = [frame[2:] for frame in self.frames]53        self.output = self.root / "live.csv"54        self.output.write_text("OLD")55        self.logfile = self.root / "status.log"5657    def tearDown(self):58        if self.proc and self.proc.poll() is None:59            self.proc.kill()60            self.proc.wait()61        self.source.close()62        self.recovery.close()63        self.temp.cleanup()6465    def start(self, *extra):66        reservation = bound()67        self.address = reservation.getsockname()68        listen = endpoint(reservation)69        reservation.close()70        args = [str(EXE), "--live", "--listen", listen, "--source", endpoint(self.source),71                "--recovery", endpoint(self.recovery), "--session", SESSION.decode(),72                "--symbol", "TEST", "--output", str(self.output), "--check-book",73                "--idle-timeout-ms", "700", "--recovery-timeout-ms", "350", "--retry-ms", "30", *extra]74        with self.logfile.open("w") as log:75            self.proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log)76        deadline = time.monotonic() + 377        while time.monotonic() < deadline:78            if "listening=" in self.logfile.read_text():79                return80            if self.proc.poll() is not None:81                self.fail(self.logfile.read_text())82            time.sleep(0.005)83        self.fail("receiver did not become ready")8485    def send(self, seq, messages=(), count=None, session=SESSION, recovery=False):86        (self.recovery if recovery else self.source).sendto(wire(seq, messages, count, session), self.address)8788    def finish(self, success=True):89        self.proc.communicate(timeout=3)90        log = self.logfile.read_text()91        self.assertEqual(self.proc.returncode, 0 if success else 1, log)92        if not success:93            self.assertEqual(self.output.read_text(), "OLD", log)94            self.assertFalse(list(self.root.glob(".live.csv.partial.*")))95        return log9697    def check_book(self):98        expected = []99        orders = {}100        for seq, timestamp, kind in [(2, 2, "A"), (3, 3, "A"), (4, 4, "X"), (5, 5, "D"), (6, 6, "D")]:101            if seq == 2:102                orders[1] = ("B", 100, 1000000)103            elif seq == 3:104                orders[2] = ("B", 200, 1000000)105            elif seq == 4:106                orders[1] = ("B", 80, 1000000)107            elif seq == 5:108                del orders[2]109            else:110                del orders[1]111            expected.append(reference_row(orders, seq, timestamp, kind, 5))112        self.assertEqual(list(csv.DictReader(io.StringIO(self.output.read_text()))), expected)113        feed = self.root / "fixture.bin"114        feed.write_bytes(b"".join(self.frames) + b"\0\0")115        baseline = subprocess.run([str(EXE), "--input", str(feed), "--symbol", "TEST", "--verify", "--check-book"],116                                  capture_output=True, check=True)117        self.assertEqual(self.output.read_bytes(), baseline.stdout)118119    def request(self):120        data, client = self.recovery.recvfrom(65536)121        self.assertEqual(data[:10], SESSION)122        self.assertEqual(len(data), 20)123        self.assertEqual(client, self.address)124        return struct.unpack(">QH", data[10:])125126    def test_clean_batched_and_duplicate(self):127        self.start()128        self.send(1, self.messages[:3])129        self.send(2, self.messages[1:4])  # overlap processed plus new message130        self.send(5, self.messages[4:])131        self.send(7, count=65535)132        log = self.finish()133        self.assertIn("duplicates=2", log)134        self.check_book()135136    def test_gap_out_of_order_duplicate_and_recovery(self):137        self.start()138        self.send(1, self.messages[:1])139        self.send(5, self.messages[4:])140        self.send(3, self.messages[2:4])141        self.send(3, self.messages[2:4])142        self.send(7, count=65535)143        seq, count = self.request()144        self.assertEqual(seq, 2)145        self.assertGreaterEqual(count, 1)146        self.send(2, self.messages[1:2], recovery=True)147        log = self.finish()148        self.assertIn("state=RECOVERING", log)149        self.assertIn("state=SYNCED", log)150        self.assertIn("nasdaq_connection=unverified", log)151        self.check_book()152153    def test_lost_first_message_and_partial_recovery(self):154        self.start()155        self.send(6, self.messages[5:])156        self.send(7, count=65535)157        first, count = self.request()158        self.assertEqual((first, count), (1, 5))159        self.send(1, self.messages[:2], recovery=True)160        first, count = self.request()161        self.assertEqual((first, count), (3, 3))162        self.send(3, self.messages[2:5], recovery=True)163        self.finish()164        self.check_book()165166    def test_heartbeat_detects_missing_tail(self):167        self.start()168        self.send(1, self.messages[:4])169        self.send(7)  # heartbeat announces sender's next sequence170        self.assertEqual(self.request(), (5, 2))171        self.send(5, self.messages[4:], recovery=True)172        self.send(7, count=65535)173        self.finish()174        self.check_book()175176    def test_end_boundary_recovers_missing_tail(self):177        self.start()178        self.send(1, self.messages[:4])179        self.send(7, count=65535)180        self.assertEqual(self.request(), (5, 2))181        self.send(5, self.messages[4:], recovery=True)182        self.finish()183        self.check_book()184185    def test_missing_end_is_not_success(self):186        self.start()187        self.send(1, self.messages)188        self.assertIn("progress timeout", self.finish(False))189190    def test_silent_socket_timeout(self):191        self.start()192        self.assertIn("progress timeout", self.finish(False))193194    def test_no_recovery_retries_and_fails(self):195        self.start()196        self.send(1, self.messages[:1])197        self.send(3, self.messages[2:])198        self.send(7, count=65535)199        self.assertEqual(self.request(), (2, 1))200        self.assertEqual(self.request(), (2, 1))201        self.assertIn("recovery timeout", self.finish(False))202203    def test_session_mismatch(self):204        self.start()205        self.send(1, self.messages[:1], session=b"OTHER00001")206        self.assertIn("session mismatch", self.finish(False))207208    def test_buffer_message_limit(self):209        self.start("--max-buffer-messages", "2")210        self.send(3, self.messages[2:])211        self.assertIn("buffer limit", self.finish(False))212213    def test_buffer_byte_limit(self):214        self.start("--max-buffer-bytes", "40")215        self.send(3, self.messages[2:4])216        self.assertIn("buffer limit", self.finish(False))217218    def test_conflicting_buffered_duplicate(self):219        self.start()220        self.send(3, self.messages[2:3])221        changed = bytearray(self.messages[2]); changed[-1] ^= 1222        self.send(3, [bytes(changed)])223        self.assertIn("conflicting buffered duplicate", self.finish(False))224225    def test_conflicting_processed_duplicate(self):226        self.start()227        self.send(1, self.messages[:2])228        changed = bytearray(self.messages[1]); changed[-1] ^= 1229        self.send(2, [bytes(changed)])230        self.assertIn("conflicting processed duplicate", self.finish(False))231232    def test_unconfigured_source_ignored(self):233        self.start()234        with bound() as stranger:235            stranger.sendto(b"garbage", self.address)236        self.send(1, self.messages)237        self.send(7, count=65535)238        self.assertIn("ignored_sources=1", self.finish())239        self.check_book()240241    def test_malformed_datagrams(self):242        cases = [b"short", wire(1, self.messages[:1])[:-1], wire(1, count=1),243                 wire(1) + b"trailing", wire(1, [b"Z"]), wire(0), wire(2**64-1),244                 wire(2**64-2, self.messages[:2]), wire(1, count=65535) + b"x"]245        for data in cases:246            with self.subTest(data=data):247                self.start()248                self.source.sendto(data, self.address)249                self.finish(False)250251    def test_invalid_book_event_fails(self):252        self.start()253        self.send(1, [self.messages[0], delete(999)[2:]])254        self.assertIn("unknown order", self.finish(False))255256    def test_inconsistent_end_boundary(self):257        self.start()258        self.send(3, self.messages[2:3])259        self.send(2, count=65535)260        self.assertIn("inconsistent end", self.finish(False))261262    def test_data_beyond_declared_end(self):263        self.start()264        self.send(4, count=65535)  # pending recovery, not complete265        self.send(4, self.messages[3:4])266        self.assertIn("beyond end", self.finish(False))267268    def test_empty_session(self):269        self.start()270        self.send(1, count=65535)271        self.assertIn("frames=0", self.finish())272        self.assertEqual(len(list(csv.DictReader(io.StringIO(self.output.read_text())))), 0)273274    def test_sender_command(self):275        self.start()276        source = endpoint(self.source); recovery = endpoint(self.recovery)277        self.source.close(); self.recovery.close()278        fixture = self.root / "sender.bin"279        sender = subprocess.run([sys.executable, str(ROOT / "tools/network_sender.py"), "--destination",280            f"{self.address[0]}:{self.address[1]}", "--source", source, "--recovery", recovery,281            "--write-fixture", str(fixture), "--serve-seconds", "0.6"], capture_output=True, text=True, timeout=3)282        self.assertEqual(sender.returncode, 0, sender.stderr)283        self.assertIn("request sequence=", sender.stdout)284        self.finish()285        self.check_book()286        self.assertEqual(fixture.read_bytes(), b"".join(self.frames) + b"\0\0")287288    def test_many_events_random_order_and_repeated_recovery(self):289        self.frames = [directory()]290        orders = {}291        expected = []292        for order in range(1, 81):293            seq = len(self.frames) + 1294            side = "B" if order % 2 else "S"295            price = 1000000 + (order % 9) * 100296            shares = 10 + order297            self.frames.append(add(order, side, shares, price, timestamp=seq))298            orders[order] = (side, shares, price)299            expected.append(reference_row(orders, seq, seq, "A", 5))300        for order in range(1, 81):301            seq = len(self.frames) + 1302            self.frames.append(reduce("X", order, 5, timestamp=seq))303            side, shares, price = orders[order]304            orders[order] = (side, shares - 5, price)305            expected.append(reference_row(orders, seq, seq, "X", 5))306        for order in range(80, 0, -1):307            seq = len(self.frames) + 1308            self.frames.append(delete(order, timestamp=seq))309            del orders[order]310            expected.append(reference_row(orders, seq, seq, "D", 5))311        self.messages = [f[2:] for f in self.frames]312        # Two messages per retry can repair the entire prefix inside the313        # recovery budget, even if source delivery is slow or packets are lost.314        self.start("--recovery-timeout-ms", "1000", "--idle-timeout-ms", "1500", "--retry-ms", "5")315        sequence = list(range(1, len(self.messages) + 1))316        random.Random(20260930).shuffle(sequence)317        missing = {1, 31, 70, 111, 190, 241}318319        def send_source():320            for seq in sequence:321                if seq not in missing:322                    self.send(seq, self.messages[seq-1:seq])323                    if seq % 19 == 0:324                        self.send(seq, self.messages[seq-1:seq])325                    time.sleep(0.0005)326            self.send(len(self.messages) + 1, count=65535)327328        # Service recovery while source traffic is still being sent. A slow329        # sender must not accidentally withhold every response until timeout.330        with ThreadPoolExecutor(max_workers=1) as sender:331            sent = sender.submit(send_source)332            self.request()  # Deliberately drop only the first response.333            deadline = time.monotonic() + 2334            while self.proc.poll() is None and time.monotonic() < deadline:335                try:336                    first, count = self.request()337                except socket.timeout:338                    continue339                self.send(first, self.messages[first-1:first-1+min(count, 2)], recovery=True)340            sent.result(timeout=3)341        log = self.finish()342        self.assertGreater(log.count("recovery_request "), 1)343        self.assertEqual(list(csv.DictReader(io.StringIO(self.output.read_text()))), expected)344        feed = self.root / "long.bin"345        feed.write_bytes(b"".join(self.frames) + b"\0\0")346        baseline = subprocess.run([str(EXE), "--input", str(feed), "--symbol", "TEST", "--verify"],347                                  check=True, capture_output=True)348        self.assertEqual(self.output.read_bytes(), baseline.stdout)349350    def test_gap_is_reported_before_any_final_publication(self):351        self.start()352        self.send(1, self.messages[:1])353        self.send(3, self.messages[2:])354        self.request()355        self.assertEqual(self.output.read_text(), "OLD")356        # Status records next=2: no later event has been applied while 2 is missing.357        self.assertIn("state=RECOVERING next_sequence=2", self.logfile.read_text())358        self.send(2, self.messages[1:2], recovery=True)359        self.send(7, count=65535)360        self.finish()361362    def test_network_options_require_live(self):363        result = subprocess.run([str(EXE), "--input", "missing.bin", "--symbol", "TEST", "--retry-ms", "100"],364                                capture_output=True, text=True)365        self.assertEqual(result.returncode, 1)366        self.assertIn("network options require --live", result.stderr)367368    def test_cli_rejects_incompatible_and_invalid_options(self):369        for args in [["--input", "x"], ["--verify"], ["--resume", "x"], ["--with-offset"],370                     ["--checkpoint-out", "x"], ["--allow-implicit-eof"], ["--retry-ms", "0"],371                     ["--session", "short"], ["--listen", "224.0.0.1:1234"],372                     ["--listen", "127.0.0.1:99999"], ["--retry-ms", "99999999999999999999"]]:373            with self.subTest(args=args):374                command = [str(EXE), "--live", "--listen", "127.0.0.1:19000", "--source", "127.0.0.1:19001",375                           "--recovery", "127.0.0.1:19002", "--session", SESSION.decode(), "--symbol", "TEST", *args]376                result = subprocess.run(command, capture_output=True, timeout=2)377                self.assertEqual(result.returncode, 1, result.stderr)378379380if __name__ == "__main__":381    unittest.main(argv=[sys.argv[0]])