m/market labResearch, with a record.
Source on GitHub ↗

test_replay.py

1"""Small, entirely synthetic BinaryFILE/ITCH 5.0 scenarios (not Nasdaq data)."""2import csv3import hashlib4import io5import random6import struct7import subprocess8import sys9import tempfile10import time11import unittest12from pathlib import Path1314from itch_fixtures import common, frame, directory, add, reduce, delete, replace, reference_row1516EXE = None17SKIPPED_LENGTHS = {18    "S": 12, "H": 25, "Y": 20, "L": 26, "V": 35, "W": 12,19    "K": 28, "J": 35, "h": 21, "P": 44, "Q": 40, "B": 19,20    "I": 50, "N": 20, "O": 48,21}222324252627282930313233343536373839404142class ReplayTests(unittest.TestCase):43    def run_feed(self, data, complete=True, extra_args=()):44        with tempfile.TemporaryDirectory() as directory_name:45            path = Path(directory_name) / "synthetic.bin"46            path.write_bytes(data + (b"\x00\x00" if complete else b""))47            return subprocess.run([str(EXE), "--input", str(path), "--symbol", "TEST", "--verify", *extra_args],48                                  text=True, capture_output=True)4950    def test_cli_groups_allow_interleaving_and_last_value_wins(self):51        data = directory() + add(1, "B", 10, 1000000) + add(2, "B", 20, 999000)52        expected = self.run_feed(data, extra_args=("--depth", "1", "--with-offset", "--check-book"))53        self.assertEqual(expected.returncode, 0, expected.stderr)54        with tempfile.TemporaryDirectory() as directory_name:55            source = Path(directory_name) / "input.bin"56            output = Path(directory_name) / "output.csv"57            ignored = Path(directory_name) / "ignored.csv"58            source.write_bytes(data + b"\x00\x00")59            result = subprocess.run([60                str(EXE), "--depth", "2", "--input", "missing.bin", "--output", str(ignored),61                "--with-offset", "--symbol", "OTHER", "--check-book", "--input", str(source),62                "--verify", "--output", str(output), "--symbol", "TEST", "--depth", "1",63            ], capture_output=True, text=True)64            self.assertEqual(result.returncode, 0, result.stderr)65            self.assertEqual(output.read_text(), expected.stdout)66            self.assertEqual(result.stdout, "")67            self.assertFalse(ignored.exists())68            self.assertIn("verified=yes", result.stderr)6970    def test_cli_missing_values_and_unknown_options_fail_before_output(self):71        value_options = (72            "--input", "--symbol", "--output", "--depth", "--checkpoint-out", "--resume",73            "--checkpoint-every-frames", "--stop-after-frame", "--listen", "--source",74            "--recovery", "--session", "--idle-timeout-ms", "--recovery-timeout-ms",75            "--retry-ms", "--max-buffer-messages", "--max-buffer-bytes", "--unknown-option",76        )77        with tempfile.TemporaryDirectory() as directory_name:78            source = Path(directory_name) / "input.bin"79            output = Path(directory_name) / "output.csv"80            source.write_bytes(directory() + b"\x00\x00")81            output.write_text("previous result")82            for option in value_options:83                with self.subTest(option=option):84                    result = subprocess.run([85                        str(EXE), "--input", str(source), "--symbol", "TEST",86                        "--output", str(output), option,87                    ], capture_output=True, text=True)88                    self.assertEqual(result.returncode, 1, result.stderr)89                    self.assertIn("usage:", result.stderr)90                    self.assertEqual(result.stdout, "")91                    self.assertEqual(output.read_text(), "previous result")92                    self.assertEqual(sorted(p.name for p in Path(directory_name).iterdir()),93                                     ["input.bin", "output.csv"])9495    def test_book_lifecycle_and_replay(self):96        data = (directory() + add(1, "B", 100, 1000000) + add(2, "B", 50, 1000000, kind="F") +97                add(3, "S", 80, 1001000) + add(4, "S", 20, 1002000) +98                reduce("E", 1, 40) + reduce("C", 3, 30) + reduce("X", 2, 20) +99                replace(1, 5, 25, 999000) + delete(3) + delete(4))100        result = self.run_feed(data)101        self.assertEqual(result.returncode, 0, result.stderr)102        rows = list(csv.DictReader(io.StringIO(result.stdout)))103        self.assertEqual(len(rows), 10)104        self.assertEqual((rows[2]["bid"], rows[2]["bid_shares"], rows[2]["ask"], rows[2]["ask_shares"]),105                         ("100.0000", "150", "100.1000", "80"))106        self.assertEqual((rows[7]["bid"], rows[7]["bid_shares"], rows[7]["bid_levels"]),107                         ("100.0000", "30", "2"))108        self.assertEqual((rows[-1]["ask"], rows[-1]["ask_levels"]), ("", "0"))109        self.assertIn("verified=yes", result.stderr)110111    def test_other_symbol_ignored(self):112        system_event = frame(common("S", 0) + b"O")113        result = self.run_feed(system_event + directory() + add(9, "B", 5, 1000000, name="OTHER", locate=8) +114                               add(1, "S", 10, 1001000))115        self.assertEqual(result.returncode, 0, result.stderr)116        self.assertEqual(len(list(csv.DictReader(io.StringIO(result.stdout)))), 1)117118    def test_optional_byte_offsets_bound_each_frame(self):119        ignored = frame(common("S", 0) + b"O")120        first = directory()121        second = add(1, "B", 5, 1000000)122        third = add(2, "S", 6, 1001000)123        feed = ignored + first + second + third124125        result = self.run_feed(feed, extra_args=("--with-offset",))126        self.assertEqual(result.returncode, 0, result.stderr)127        rows = list(csv.DictReader(io.StringIO(result.stdout)))128        self.assertEqual(len(rows), 2)129        self.assertEqual(list(rows[0]), ["frame", "timestamp_ns", "type", "bid",130                                         "bid_shares", "ask", "ask_shares", "bid_levels",131                                         "ask_levels", "bid_depth", "ask_depth",132                                         "byte_offset", "byte_end_offset"])133        self.assertEqual([row["byte_offset"] for row in rows],134                         [str(len(ignored + first)), str(len(ignored + first + second))])135        self.assertEqual([row["byte_end_offset"] for row in rows],136                         [str(len(ignored + first + second)), str(len(feed))])137        for row in rows:138            offset = int(row["byte_offset"])139            length = struct.unpack_from(">H", feed, offset)[0]140            self.assertEqual(int(row["byte_end_offset"]), offset + 2 + length)141            self.assertEqual(chr(feed[offset + 2]), row["type"])142            self.assertIn(length, (36, 40))143144        default = self.run_feed(feed)145        self.assertEqual(default.returncode, 0, default.stderr)146        self.assertNotIn("byte_offset", next(csv.reader(io.StringIO(default.stdout))))147148    def test_every_skipped_type_is_length_checked(self):149        for kind, length in SKIPPED_LENGTHS.items():150            with self.subTest(type=kind):151                valid = frame(kind.encode() + bytes(length - 1))152                result = self.run_feed(directory() + valid + add(1, "B", 5, 1000000))153                self.assertEqual(result.returncode, 0, result.stderr)154                self.assertEqual(len(list(csv.DictReader(io.StringIO(result.stdout)))), 1)155                self.assertIn("frames=3", result.stderr)156157                invalid = frame(kind.encode() + bytes(length - 2))158                result = self.run_feed(directory() + invalid)159                self.assertNotEqual(result.returncode, 0)160                self.assertIn(f"frame 2: invalid {kind} length", result.stderr)161162    def test_every_book_message_type_is_length_checked(self):163        valid_frames = [164            directory(), add(1, "B", 5, 1000000), add(1, "B", 5, 1000000, kind="F"),165            reduce("E", 1, 1), reduce("C", 1, 1), reduce("X", 1, 1),166            delete(1), replace(1, 2, 5, 1000000),167        ]168        for valid in valid_frames:169            kind = chr(valid[2])170            with self.subTest(type=kind):171                shortened = frame(valid[2:-1])172                result = self.run_feed(shortened)173                self.assertNotEqual(result.returncode, 0)174                self.assertIn(f"frame 1: invalid {kind} length", result.stderr)175176    def test_later_corruption_reports_first_bad_frame(self):177        prefix = directory() + add(1, "B", 10, 1000000)178        cases = [179            (prefix + b"\x00", "truncated frame length after frame 2"),180            (prefix + b"\x00\x24A", "frame 3: truncated payload"),181            (prefix + frame(b"A"), "frame 3: invalid A length"),182            (prefix + frame(b"Z"), "frame 3: unknown ITCH type"),183            (prefix, "missing BinaryFILE end marker after frame 2"),184            (prefix + b"\x00\x00X", "bytes after BinaryFILE end marker after frame 2"),185            (prefix + directory(name="TEST", locate=8), "frame 3: symbol changed stock locate"),186            (prefix + directory(name="OTHER", locate=7), "frame 3: stock locate belongs to another symbol"),187        ]188        for data, error in cases:189            with self.subTest(error=error):190                result = self.run_feed(data, complete=False)191                self.assertNotEqual(result.returncode, 0)192                self.assertIn(error, result.stderr)193194    def test_implicit_eof_requires_explicit_option_and_final_end_of_messages(self):195        end_messages = frame(common("S", 0) + b"C")196        prefix = directory() + add(1, "B", 10, 1000000)197        strict = self.run_feed(prefix + end_messages, complete=False)198        self.assertNotEqual(strict.returncode, 0)199        self.assertIn("missing BinaryFILE end marker after frame 3", strict.stderr)200201        allowed = self.run_feed(prefix + end_messages, complete=False,202                                extra_args=("--allow-implicit-eof",))203        self.assertEqual(allowed.returncode, 0, allowed.stderr)204        self.assertIn("frames=3", allowed.stderr)205        self.assertEqual(len(list(csv.DictReader(io.StringIO(allowed.stdout)))), 1)206207        for bad in (prefix, prefix + end_messages + add(2, "S", 2, 1001000),208                    prefix + end_messages + b"\x00", prefix + frame(common("S", 7) + b"C")):209            with self.subTest(bad=bad[-16:]):210                result = self.run_feed(bad, complete=False,211                                       extra_args=("--allow-implicit-eof",))212                self.assertNotEqual(result.returncode, 0)213214    def test_output_file_is_published_only_on_success(self):215        with tempfile.TemporaryDirectory() as directory_name:216            root = Path(directory_name)217            source = root / "source.bin"218            destination = root / "book.csv"219            command = [str(EXE), "--input", str(source), "--symbol", "TEST",220                       "--output", str(destination), "--verify"]221            valid_prefix = directory() + add(1, "B", 10, 1000000)222223            destination.write_text("previous complete result\n")224            source.write_bytes(valid_prefix + frame(b"Z") + b"\x00\x00")225            failed = subprocess.run(command, text=True, capture_output=True)226            self.assertNotEqual(failed.returncode, 0)227            self.assertIn("frame 3: unknown ITCH type", failed.stderr)228            self.assertEqual(destination.read_text(), "previous complete result\n")229            self.assertEqual(list(root.glob(".book.csv.partial.*")), [])230231            destination.unlink()232            failed_without_old_file = subprocess.run(command, text=True, capture_output=True)233            self.assertNotEqual(failed_without_old_file.returncode, 0)234            self.assertFalse(destination.exists())235            self.assertEqual(list(root.glob(".book.csv.partial.*")), [])236237            source.write_bytes(valid_prefix + b"\x00\x00")238            succeeded = subprocess.run(command, text=True, capture_output=True)239            self.assertEqual(succeeded.returncode, 0, succeeded.stderr)240            rows = list(csv.DictReader(io.StringIO(destination.read_text())))241            self.assertEqual(len(rows), 1)242            self.assertEqual(rows[0]["bid_depth"], "100.0000:10")243            self.assertEqual(list(root.glob(".book.csv.partial.*")), [])244245            published = destination.read_text()246            destination.write_text("older result\n")247            replacement = subprocess.run(command, text=True, capture_output=True)248            self.assertEqual(replacement.returncode, 0, replacement.stderr)249            self.assertEqual(destination.read_text(), published)250251            original_input = source.read_bytes()252            same_file = subprocess.run([str(EXE), "--input", str(source), "--symbol", "TEST",253                                        "--output", str(source)], text=True, capture_output=True)254            self.assertNotEqual(same_file.returncode, 0)255            self.assertIn("input and output refer to the same file", same_file.stderr)256            self.assertEqual(source.read_bytes(), original_input)257258    def checkpoint_command(self, source, *args):259        return subprocess.run([str(EXE), "--input", str(source), "--symbol", "TEST", *args],260                              text=True, capture_output=True)261262    def checkpoint_layout(self, data):263        """Independent parser for v1/v2 metadata and mutation positions."""264        body = data[:-64]265        self.assertEqual(hashlib.sha256(body).hexdigest().encode(), data[-64:])266        self.assertEqual(body[:8], b"ITCHCP01")267        version = struct.unpack_from(">I", body, 8)[0]268        self.assertIn(version, (1, 2))269        pos = 12270        fields = {}271        for key in ("size", "digest", "symbol", "next_frame", "offset", "selected", "hash", "depth"):272            fields[key] = pos273            n = struct.unpack_from(">Q", body, pos)[0]274            pos += 8275            if key in ("digest", "symbol"):276                fields[key + "_value"] = body[pos:pos+n]277                pos += n278            else:279                fields[key + "_value"] = n280        for key in ("with_offset", "implicit_eof", "last_end", "has_locate"):281            fields[key] = pos282            pos += 1283        fields["locate"] = pos284        pos += 2285        fields["order_count"] = pos286        count = struct.unpack_from(">Q", body, pos)[0]287        pos += 8288        fields["orders"] = pos289        pos += count * 17290        fields["bid_count"] = pos291        count = struct.unpack_from(">Q", body, pos)[0]292        fields["bids"] = pos + 8293        pos += 8 + count * 12294        fields["ask_count"] = pos295        fields["asks"] = pos + 8296        pos += 8 + struct.unpack_from(">Q", body, pos)[0] * 12297        fields["version"] = version298        if version == 2:299            fields["csv_binding"] = pos300            count = struct.unpack_from(">Q", body, pos)[0]301            fields["csv_path_value"] = body[pos+8:pos+8+count].decode()302            pos += 8 + count303            fields["csv_bytes"] = pos304            fields["csv_bytes_value"] = struct.unpack_from(">Q", body, pos)[0]305        return fields306307    def test_checkpoint_suffix_and_final_state_at_every_boundary(self):308        payloads = [frame(common("S", 0) + b"O"), directory(),309                    add(90, "B", 3, 900000, name="OTHER", locate=8),310                    add(1, "B", 100, 1000000), add(2, "B", 50, 1000000, kind="F"),311                    add(3, "S", 80, 1001000), add(4, "S", 20, 1002000),312                    reduce("E", 1, 40), reduce("C", 3, 30), reduce("X", 2, 50),313                    replace(1, 5, 25, 999000), delete(3), delete(4), delete(5),314                    add(6, "B", 11, 990000), add(7, "S", 12, 1010000),315                    add(8, "B", 13, 990000), frame(common("S", 0) + b"C")]316        with tempfile.TemporaryDirectory() as directory_name:317            root = Path(directory_name)318            source, checkpoint, full_end, resumed_end = [root / n for n in319                ("feed.bin", "saved.cp", "full.cp", "resumed.cp")]320            for flags, marker in (((), b"\0\0"),321                    (("--depth", "1", "--with-offset", "--allow-implicit-eof"), b"")):322                source.write_bytes(b"".join(payloads) + marker)323                full = self.checkpoint_command(source, *flags, "--verify", "--check-book",324                                               "--checkpoint-out", str(full_end))325                self.assertEqual(full.returncode, 0, full.stderr)326                lines = full.stdout.splitlines()327                for boundary in range(len(payloads) + 1):328                    with self.subTest(flags=flags, boundary=boundary):329                        prefix = self.checkpoint_command(source, *flags, "--verify", "--check-book",330                            "--stop-after-frame", str(boundary), "--checkpoint-out", str(checkpoint))331                        self.assertEqual(prefix.returncode, 0, prefix.stderr)332                        self.assertIn("scope=prefix complete=no", prefix.stderr)333                        layout = self.checkpoint_layout(checkpoint.read_bytes())334                        self.assertEqual(layout["offset_value"], sum(map(len, payloads[:boundary])))335                        self.assertEqual(layout["next_frame_value"], boundary + 1)336                        self.assertEqual(layout["digest_value"], hashlib.sha256(source.read_bytes()).hexdigest().encode())337                        resumed = self.checkpoint_command(source, *flags, "--verify", "--check-book",338                            "--resume", str(checkpoint), "--checkpoint-out", str(resumed_end))339                        self.assertEqual(resumed.returncode, 0, resumed.stderr)340                        self.assertIn("scope=suffix complete=yes", resumed.stderr)341                        expected = [lines[0]] + [line for line in lines[1:] if int(line.split(",")[0]) > boundary]342                        self.assertEqual(resumed.stdout.splitlines(), expected)343                        self.assertEqual(prefix.stdout.splitlines()[1:] + resumed.stdout.splitlines()[1:], lines[1:])344                        # Equal bytes cover every final order, level, locate, counter and cumulative CSV hash.345                        self.assertEqual(resumed_end.read_bytes(), full_end.read_bytes())346                terminal = self.checkpoint_command(source, *flags, "--resume", str(full_end))347                self.assertEqual(terminal.returncode, 0, terminal.stderr)348                self.assertEqual(terminal.stdout.splitlines(), [lines[0]])349350    def test_checkpoint_chained_resume_and_empty_session(self):351        with tempfile.TemporaryDirectory() as directory_name:352            root = Path(directory_name)353            source, saved = root / "feed.bin", root / "saved.cp"354            source.write_bytes(directory() + add(1, "B", 10, 1000000) +355                               replace(1, 2, 5, 999000) + delete(2) + b"\0\0")356            full = self.checkpoint_command(source)357            first = self.checkpoint_command(source, "--stop-after-frame", "2", "--checkpoint-out", str(saved))358            self.assertEqual(first.returncode, 0, first.stderr)359            middle = self.checkpoint_command(source, "--resume", str(saved), "--stop-after-frame", "3",360                                             "--checkpoint-out", str(saved), "--verify")361            self.assertEqual(middle.returncode, 0, middle.stderr)362            last = self.checkpoint_command(source, "--resume", str(saved))363            self.assertEqual(last.returncode, 0, last.stderr)364            self.assertEqual(first.stdout.splitlines() + middle.stdout.splitlines()[1:] +365                             last.stdout.splitlines()[1:], full.stdout.splitlines())366            source.write_bytes(b"\0\0")367            empty = self.checkpoint_command(source, "--checkpoint-out", str(saved), "--verify")368            self.assertEqual(empty.returncode, 0, empty.stderr)369            restored = self.checkpoint_command(source, "--resume", str(saved), "--verify")370            self.assertEqual(restored.returncode, 0, restored.stderr)371            self.assertEqual(empty.stdout, restored.stdout)372373    def test_periodic_checkpoint_saves_skipped_frames_before_error(self):374        with tempfile.TemporaryDirectory() as directory_name:375            root = Path(directory_name)376            source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"377            good = directory() + frame(common("S", 0) + b"O")378            source.write_bytes(good + frame(b"Z") + b"\0\0")379            output.write_text("previous completed CSV")380            for with_csv in (False, True):381                args = ("--output", str(output)) if with_csv else ()382                result = self.checkpoint_command(source, "--checkpoint-every-frames", "2",383                    "--checkpoint-out", str(saved), *args)384                self.assertNotEqual(result.returncode, 0)385                self.assertIn("frame 3: unknown", result.stderr)386                layout = self.checkpoint_layout(saved.read_bytes())387                self.assertEqual(layout["next_frame_value"], 3)388                self.assertEqual(layout["offset_value"], len(good))389                self.assertEqual(layout["selected_value"], 0)390                self.assertEqual(layout["version"], 2 if with_csv else 1)391                if with_csv:392                    journal = Path(layout["csv_path_value"])393                    self.assertTrue(journal.exists())394                    self.assertEqual(journal.stat().st_size, layout["csv_bytes_value"])395                self.assertEqual(output.read_text(), "previous completed CSV")396397    def test_resume_csv_matches_full_replay_and_supports_multiple_stops(self):398        payloads = [directory(), add(1, "B", 10, 1000000),399                    frame(common("W", 0) + b"0"), add(2, "S", 20, 1001000),400                    replace(1, 3, 5, 999000), reduce("C", 2, 10), delete(3)]401        with tempfile.TemporaryDirectory() as directory_name:402            root = Path(directory_name)403            source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"404            source.write_bytes(b"".join(payloads) + b"\0\0")405            flags = ("--with-offset", "--depth", "1", "--verify", "--check-book")406            full = self.checkpoint_command(source, *flags)407            self.assertEqual(full.returncode, 0, full.stderr)408            for boundary in range(len(payloads) + 1):409                first = self.checkpoint_command(source, *flags, "--stop-after-frame", str(boundary),410                    "--checkpoint-every-frames", "2", "--checkpoint-out", str(saved), "--output", str(output))411                self.assertEqual(first.returncode, 0, first.stderr)412                layout = self.checkpoint_layout(saved.read_bytes())413                self.assertEqual(layout["version"], 2)414                # Uncheckpointed bytes, even complete duplicate rows, must be discarded.415                with Path(layout["csv_path_value"]).open("ab") as journal:416                    journal.write(b"uncheckpointed partial row,123")417                second = self.checkpoint_command(source, *flags, "--resume", str(saved), "--resume-csv",418                    "--output", str(output), "--checkpoint-every-frames", "2")419                self.assertEqual(second.returncode, 0, second.stderr)420                self.assertIn("scope=full complete=yes", second.stderr)421                self.assertEqual(output.read_text(), full.stdout)422                again = self.checkpoint_command(source, *flags, "--resume", str(saved), "--resume-csv",423                                                "--output", str(output))424                self.assertEqual(again.returncode, 0, again.stderr)425                self.assertEqual(output.read_text(), full.stdout)426            # Chained stops retain the full prefix, not just the newest suffix.427            first = self.checkpoint_command(source, *flags, "--stop-after-frame", "2",428                                           "--checkpoint-out", str(saved), "--output", str(output))429            self.assertEqual(first.returncode, 0, first.stderr)430            for boundary in (4, 6):431                part = self.checkpoint_command(source, *flags, "--resume", str(saved), "--resume-csv",432                    "--output", str(output), "--stop-after-frame", str(boundary))433                self.assertEqual(part.returncode, 0, part.stderr)434                self.assertIn("scope=prefix complete=no", part.stderr)435            end = self.checkpoint_command(source, *flags, "--resume", str(saved), "--resume-csv",436                                         "--output", str(output))437            self.assertEqual(end.returncode, 0, end.stderr)438            self.assertEqual(output.read_text(), full.stdout)439440    def test_resume_csv_rejects_missing_or_corrupt_prefix_without_overwriting(self):441        with tempfile.TemporaryDirectory() as directory_name:442            root = Path(directory_name)443            source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"444            source.write_bytes(directory() + add(1, "B", 10, 1000000) + delete(1) + b"\0\0")445            first = self.checkpoint_command(source, "--stop-after-frame", "2", "--checkpoint-out", str(saved),446                                           "--output", str(output))447            self.assertEqual(first.returncode, 0, first.stderr)448            cp = saved.read_bytes()449            layout = self.checkpoint_layout(cp)450            journal = Path(layout["csv_path_value"])451            original = journal.read_bytes()452            for damaged, error in ((original[:-1], "truncated CSV prefix"),453                    (b"x" + original[1:], "header mismatch"),454                    (original.replace(b"100.0000:10", b"100.0000:11"), "hash mismatch")):455                journal.write_bytes(damaged)456                output.write_text("previous full CSV")457                result = self.checkpoint_command(source, "--resume", str(saved), "--resume-csv", "--output", str(output))458                self.assertNotEqual(result.returncode, 0)459                self.assertIn(error, result.stderr)460                self.assertEqual(saved.read_bytes(), cp)461                self.assertEqual(output.read_text(), "previous full CSV")462            journal.unlink()463            result = self.checkpoint_command(source, "--resume", str(saved), "--resume-csv", "--output", str(output))464            self.assertNotEqual(result.returncode, 0)465            self.assertIn("cannot open saved CSV prefix", result.stderr)466            self.assertEqual(saved.read_bytes(), cp)467            state_only = self.checkpoint_command(source, "--checkpoint-out", str(saved))468            self.assertEqual(state_only.returncode, 0, state_only.stderr)469            result = self.checkpoint_command(source, "--resume", str(saved), "--resume-csv", "--output", str(output))470            self.assertNotEqual(result.returncode, 0)471            self.assertIn("no saved CSV prefix", result.stderr)472473    def test_periodic_checkpoint_recovers_after_process_is_killed(self):474        with tempfile.TemporaryDirectory() as directory_name:475            root = Path(directory_name)476            source, saved, output, expected = [root / name for name in ("feed.bin", "saved.cp", "out.csv", "full.csv")]477            source.write_bytes(directory() + (add(1, "B", 10, 1000000) + delete(1)) * 200000 + b"\0\0")478            output.write_text("previous completed CSV")479            command = [str(EXE), "--input", str(source), "--symbol", "TEST", "--output", str(output),480                       "--checkpoint-out", str(saved), "--checkpoint-every-frames", "1000"]481            process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)482            try:483                deadline = time.monotonic() + 15484                while not saved.exists() and process.poll() is None and time.monotonic() < deadline:485                    time.sleep(0.005)486                self.assertTrue(saved.exists(), "no periodic checkpoint published")487                self.assertIsNone(process.poll(), "replay completed before interruption")488                process.kill()489                process.communicate(timeout=5)490            finally:491                if process.poll() is None:492                    process.kill()493                    process.communicate(timeout=5)494            layout = self.checkpoint_layout(saved.read_bytes())495            self.assertLess(layout["next_frame_value"], 400002)496            self.assertEqual(output.read_text(), "previous completed CSV")497            resumed = self.checkpoint_command(source, "--resume", str(saved), "--resume-csv",498                                              "--output", str(output), "--verify")499            self.assertEqual(resumed.returncode, 0, resumed.stderr)500            full = self.checkpoint_command(source, "--output", str(expected))501            self.assertEqual(full.returncode, 0, full.stderr)502            self.assertEqual(hashlib.sha256(output.read_bytes()).digest(), hashlib.sha256(expected.read_bytes()).digest())503            self.assertEqual(output.stat().st_size, expected.stat().st_size)504505    def test_checkpoint_save_rules_do_not_depend_on_csv_mode(self):506        with tempfile.TemporaryDirectory() as directory_name:507            root = Path(directory_name)508            source, saved, other, output = [root / name for name in509                ("feed.bin", "saved.cp", "other.cp", "out.csv")]510            source.write_bytes(directory() + add(1, "B", 10, 1000000) +511                               add(2, "S", 20, 1001000) + delete(1) +512                               frame(common("S", 0) + b"O") + delete(2) + b"\0\0")513            full = self.checkpoint_command(source)514            created = self.checkpoint_command(source, "--stop-after-frame", "3", "--checkpoint-out", str(saved),515                                              "--output", str(output))516            self.assertEqual(created.returncode, 0, created.stderr)517            original = saved.read_bytes()518            suffix = self.checkpoint_command(source, "--resume", str(saved))519            self.assertEqual(suffix.returncode, 0, suffix.stderr)520            self.assertEqual(saved.read_bytes(), original)521            csv_only = self.checkpoint_command(source, "--resume", str(saved), "--resume-csv",522                                                "--output", str(output), "--verify")523            self.assertEqual(csv_only.returncode, 0, csv_only.stderr)524            self.assertEqual(output.read_text(), full.stdout)525            self.assertEqual(saved.read_bytes(), original, "CSV continuation must not enable saving")526            # Both CSV modes use the same inferred destination when saving is requested.527            modes = ((False, ()), (False, ("--output", str(output))),528                     (True, ("--resume-csv", "--output", str(output))))529            for csv_mode, extra in modes:530                saved.write_bytes(original)531                automatic = self.checkpoint_command(source, "--resume", str(saved),532                    "--checkpoint-every-frames", "2", "--verify", *extra)533                self.assertEqual(automatic.returncode, 0, automatic.stderr)534                if extra:535                    expected = full.stdout.splitlines() if csv_mode else [full.stdout.splitlines()[0]] + [536                        row for row in full.stdout.splitlines()[1:] if int(row.split(",")[0]) > 3]537                    self.assertEqual(output.read_text().splitlines(), expected)538                self.assertEqual(self.checkpoint_layout(saved.read_bytes())["next_frame_value"], 7)539                saved.write_bytes(original)540                explicit = self.checkpoint_command(source, "--resume", str(saved), "--checkpoint-out", str(other),541                    "--checkpoint-every-frames", "2", *extra)542                self.assertEqual(explicit.returncode, 0, explicit.stderr)543                self.assertEqual(saved.read_bytes(), original, "explicit destination must take precedence")544                self.assertEqual(self.checkpoint_layout(other.read_bytes())["next_frame_value"], 7)545                saved.write_bytes(original)546                stopped = self.checkpoint_command(source, "--resume", str(saved), "--stop-after-frame", "4", *extra)547                self.assertEqual(stopped.returncode, 0, stopped.stderr)548                self.assertEqual(self.checkpoint_layout(saved.read_bytes())["next_frame_value"], 5)549550    def test_inferred_periodic_destination_is_written_before_later_failure(self):551        with tempfile.TemporaryDirectory() as directory_name:552            root = Path(directory_name)553            source, saved = root / "feed.bin", root / "saved.cp"554            source.write_bytes(directory() + add(1, "B", 10, 1000000) +555                               frame(common("S", 0) + b"O") + delete(1) + frame(b"Z") + b"\0\0")556            first = self.checkpoint_command(source, "--checkpoint-out", str(saved), "--stop-after-frame", "3")557            self.assertEqual(first.returncode, 0, first.stderr)558            resumed = self.checkpoint_command(source, "--resume", str(saved), "--checkpoint-every-frames", "2")559            self.assertNotEqual(resumed.returncode, 0)560            self.assertIn("frame 5: unknown", resumed.stderr)561            state = self.checkpoint_layout(saved.read_bytes())562            self.assertEqual(state["next_frame_value"], 5)563            self.assertEqual(struct.unpack_from(">Q", saved.read_bytes(), state["order_count"])[0], 0)564565    def test_periodic_checkpoint_options_are_explicit_and_validated(self):566        for args, error in ((("--checkpoint-every-frames", "0"), "must be positive"),567                (("--checkpoint-every-frames", "-1"), "must be"),568                (("--checkpoint-every-frames", "abc"), "must be"),569                (("--checkpoint-every-frames", "18446744073709551615"), "must be"),570                (("--checkpoint-every-frames", "10"), "requires --checkpoint-out"),571                (("--resume-csv",), "requires --resume and --output")):572            result = self.run_feed(directory(), extra_args=args)573            self.assertNotEqual(result.returncode, 0)574            self.assertIn(error, result.stderr)575        with tempfile.TemporaryDirectory() as directory_name:576            root = Path(directory_name)577            source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"578            source.write_bytes(directory() + b"\0\0")579            first = self.checkpoint_command(source, "--checkpoint-out", str(saved), "--output", str(output))580            self.assertEqual(first.returncode, 0, first.stderr)581            layout = self.checkpoint_layout(saved.read_bytes())582            journal = Path(layout["csv_path_value"])583            original = journal.read_bytes()584            for args in (("--output", str(journal), "--resume-csv"),585                         ("--checkpoint-out", str(journal))):586                result = self.checkpoint_command(source, "--resume", str(saved), *args)587                self.assertNotEqual(result.returncode, 0)588                self.assertIn("must be distinct", result.stderr)589                self.assertEqual(journal.read_bytes(), original)590            result = self.checkpoint_command(source, "--resume", str(saved), "--output", str(output),591                "--checkpoint-out", str(saved), "--checkpoint-every-frames", "1")592            self.assertEqual(result.returncode, 0, result.stderr)593            self.assertEqual(self.checkpoint_layout(saved.read_bytes())["version"], 1)594            self.assertEqual(len(output.read_text().splitlines()), 1)595596    def test_sha256_padding_and_streaming_against_hashlib(self):597        with tempfile.TemporaryDirectory() as directory_name:598            source, saved = Path(directory_name) / "feed.bin", Path(directory_name) / "saved.cp"599            for size in (0, 1, 55, 56, 63, 64, 65, 127, 128, 1000000):600                with self.subTest(size=size):601                    source.write_bytes(bytes(i % 251 for i in range(size)))602                    result = self.checkpoint_command(source, "--stop-after-frame", "0", "--checkpoint-out", str(saved))603                    self.assertEqual(result.returncode, 0, result.stderr)604                    layout = self.checkpoint_layout(saved.read_bytes())605                    self.assertEqual(layout["size_value"], size)606                    self.assertEqual(layout["digest_value"], hashlib.sha256(source.read_bytes()).hexdigest().encode())607608    def test_checkpoint_corruption_and_semantic_validation_preserve_outputs(self):609        with tempfile.TemporaryDirectory() as directory_name:610            root = Path(directory_name)611            source, saved, output, new_cp = [root / n for n in ("feed.bin", "saved.cp", "book.csv", "new.cp")]612            source.write_bytes(directory() + add(1, "B", 10, 1000000) +613                               add(2, "B", 20, 1000000) + delete(1) + b"\0\0")614            created = self.checkpoint_command(source, "--stop-after-frame", "3", "--checkpoint-out", str(saved))615            self.assertEqual(created.returncode, 0, created.stderr)616            original = saved.read_bytes()617            f = self.checkpoint_layout(original)618619            def changed(offset, value):620                body = bytearray(original[:-64])621                body[offset:offset+len(value)] = value622                return bytes(body) + hashlib.sha256(body).hexdigest().encode()623624            cases = [625                (original[:-1], "checksum"), (b"broken", "truncated"),626                (original[:30] + bytes([original[30] ^ 1]) + original[31:], "checksum"),627                (changed(0, b"INVALID!"), "magic"),628                (changed(8, struct.pack(">I", 3)), "version"),629                (changed(f["offset"], struct.pack(">Q", f["offset_value"]-1)), "frame boundary"),630                (changed(f["next_frame"], struct.pack(">Q", 3)), "frame/offset"),631                (changed(f["last_end"], b"\1"), "end-message"),632                (changed(f["with_offset"], b"\2"), "invalid flag"),633                (changed(f["orders"] + 8, b"Q"), "invalid live order"),634                (changed(f["orders"] + 9, bytes(4)), "invalid live order"),635                (changed(f["orders"] + 17, original[f["orders"]:f["orders"]+8]), "duplicate order"),636                (changed(f["bids"] + 4, struct.pack(">Q", 999)), "price levels"),637                (changed(f["order_count"], struct.pack(">Q", 2**64-1)), "order count"),638                (changed(f["depth"], struct.pack(">Q", 1001)), "invalid depth"),639            ]640            for damaged, reason in cases:641                with self.subTest(reason=reason):642                    saved.write_bytes(damaged)643                    output.write_text("previous CSV")644                    new_cp.write_bytes(b"previous checkpoint")645                    result = self.checkpoint_command(source, "--resume", str(saved),646                        "--output", str(output), "--checkpoint-out", str(new_cp), "--verify")647                    self.assertNotEqual(result.returncode, 0)648                    self.assertIn(reason, result.stderr)649                    self.assertEqual(output.read_text(), "previous CSV")650                    self.assertEqual(new_cp.read_bytes(), b"previous checkpoint")651                    self.assertEqual(list(root.glob(".*.partial.*")), [])652653            saved.write_bytes(original)654            for args, reason in ((("--symbol", "OTHER"), "symbol mismatch"),655                    (("--depth", "2"), "options mismatch"),656                    (("--with-offset",), "options mismatch"),657                    (("--allow-implicit-eof",), "options mismatch")):658                result = self.checkpoint_command(source, "--resume", str(saved), "--output", str(output), *args)659                self.assertNotEqual(result.returncode, 0)660                self.assertIn(reason, result.stderr)661                self.assertEqual(output.read_text(), "previous CSV")662            # Same-size change beyond the checkpoint must still be detected.663            source.write_bytes(source.read_bytes()[:-1] + b"X")664            result = self.checkpoint_command(source, "--resume", str(saved), "--output", str(output))665            self.assertNotEqual(result.returncode, 0)666            self.assertIn("input SHA-256 mismatch", result.stderr)667            self.assertEqual(output.read_text(), "previous CSV")668669    def test_checkpoint_bad_suffix_points_to_untrusted_byte(self):670        with tempfile.TemporaryDirectory() as directory_name:671            root = Path(directory_name)672            source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"673            prefix = directory() + add(1, "B", 10, 1000000)674            source.write_bytes(prefix + frame(b"Z") + b"\0\0")675            created = self.checkpoint_command(source, "--stop-after-frame", "2", "--checkpoint-out", str(saved))676            self.assertEqual(created.returncode, 0, created.stderr)677            original = saved.read_bytes()678            output.write_text("previous")679            result = self.checkpoint_command(source, "--resume", str(saved), "--output", str(output),680                                             "--checkpoint-out", str(saved))681            self.assertNotEqual(result.returncode, 0)682            self.assertIn("frame 3: unknown ITCH type", result.stderr)683            self.assertIn(f"first untrusted byte_offset={len(prefix)}", result.stderr)684            self.assertEqual(saved.read_bytes(), original)685            self.assertEqual(output.read_text(), "previous")686687    def test_checkpoint_argument_and_path_validation(self):688        for value in ("-1", "1x", "18446744073709551615", "18446744073709551616"):689            result = self.run_feed(directory(), extra_args=("--stop-after-frame", value))690            self.assertNotEqual(result.returncode, 0)691            self.assertIn("--stop-after-frame must be", result.stderr)692        result = self.run_feed(directory(), extra_args=("--stop-after-frame", "0"))693        self.assertNotEqual(result.returncode, 0)694        self.assertIn("requires --checkpoint-out", result.stderr)695        with tempfile.TemporaryDirectory() as directory_name:696            root = Path(directory_name)697            source, saved, output = root / "feed.bin", root / "saved.cp", root / "out.csv"698            source.write_bytes(directory() + b"\0\0")699            saved.write_bytes(b"previous")700            result = self.checkpoint_command(source, "--stop-after-frame", "2", "--checkpoint-out", str(saved))701            self.assertNotEqual(result.returncode, 0)702            self.assertIn("exceeds input", result.stderr)703            self.assertEqual(saved.read_bytes(), b"previous")704            for args in (("--checkpoint-out", str(source)),705                         ("--checkpoint-out", str(output), "--output", str(output)),706                         ("--resume", str(saved), "--output", str(saved))):707                result = self.checkpoint_command(source, *args)708                self.assertNotEqual(result.returncode, 0)709                self.assertIn("must be distinct", result.stderr)710            alias = root / "alias.cp"711            alias.symlink_to(source)712            result = self.checkpoint_command(source, "--checkpoint-out", str(alias))713            self.assertNotEqual(result.returncode, 0)714            self.assertEqual(source.read_bytes(), directory() + b"\0\0")715716    def test_every_event_matches_independent_reference(self):717        rng = random.Random(20260926)718        payloads = [directory()]719        orders = {}720        expected = {2: [], 5: []}721        next_id = 1722        prices = [990000, 995000, 1000000, 1001000, 1002000, 1010000, 1020000]723724        def record(payload, kind, timestamp):725            payloads.append(payload)726            for limit in expected:727                expected[limit].append(reference_row(orders, len(payloads), timestamp, kind, limit))728729        # Multiple orders share prices; more than five levels exist on both sides.730        for side in ("B", "S"):731            for raw_price in prices:732                for _ in range(2):733                    order_id = next_id734                    next_id += 1735                    shares = rng.randint(1, 500)736                    orders[order_id] = (side, shares, raw_price)737                    kind = "F" if order_id % 3 == 0 else "A"738                    record(add(order_id, side, shares, raw_price, kind=kind,739                               timestamp=next_id), kind, next_id)740741        for step in range(250):742            timestamp = 1000 + step743            action = rng.choice(("add", "reduce", "delete", "replace"))744            if not orders:745                action = "add"746            if action == "add":747                order_id = next_id748                next_id += 1749                side = rng.choice(("B", "S"))750                shares = rng.randint(1, 500)751                raw_price = rng.choice(prices)752                orders[order_id] = (side, shares, raw_price)753                kind = rng.choice(("A", "F"))754                record(add(order_id, side, shares, raw_price, kind=kind, timestamp=timestamp),755                       kind, timestamp)756            else:757                old_id = rng.choice(tuple(orders))758                side, old_shares, raw_price = orders[old_id]759                if action == "reduce":760                    removed = rng.randint(1, old_shares)761                    kind = rng.choice(("E", "C", "X"))762                    if removed == old_shares:763                        del orders[old_id]764                    else:765                        orders[old_id] = (side, old_shares - removed, raw_price)766                    record(reduce(kind, old_id, removed, timestamp=timestamp), kind, timestamp)767                elif action == "delete":768                    del orders[old_id]769                    record(delete(old_id, timestamp=timestamp), "D", timestamp)770                else:771                    new_id = next_id772                    next_id += 1773                    new_shares = rng.randint(1, 500)774                    new_price = rng.choice(prices)775                    del orders[old_id]776                    orders[new_id] = (side, new_shares, new_price)777                    record(replace(old_id, new_id, new_shares, new_price,778                                   timestamp=timestamp), "U", timestamp)779780        data = b"".join(payloads)781        for limit, rows_expected in expected.items():782            with self.subTest(depth=limit):783                result = self.run_feed(data, extra_args=("--depth", str(limit), "--check-book"))784                self.assertEqual(result.returncode, 0, result.stderr)785                rows_actual = list(csv.DictReader(io.StringIO(result.stdout)))786                self.assertEqual(rows_actual, rows_expected)787                self.assertIn("verified=yes", result.stderr)788                # Restored books must also agree with the independent order model.789                with tempfile.TemporaryDirectory() as directory_name:790                    source = Path(directory_name) / "random.bin"791                    checkpoint = Path(directory_name) / "random.cp"792                    source.write_bytes(data + b"\0\0")793                    for boundary in (5, 63, 190, len(payloads)):794                        saved = self.checkpoint_command(source, "--depth", str(limit), "--check-book",795                            "--stop-after-frame", str(boundary), "--checkpoint-out", str(checkpoint))796                        self.assertEqual(saved.returncode, 0, saved.stderr)797                        restored = self.checkpoint_command(source, "--depth", str(limit), "--check-book",798                            "--resume", str(checkpoint), "--verify")799                        self.assertEqual(restored.returncode, 0, restored.stderr)800                        self.assertEqual(list(csv.DictReader(io.StringIO(restored.stdout))),801                            [row for row in rows_expected if int(row["frame"]) > boundary])802803    def test_empty_and_single_level_depth(self):804        data = directory() + add(1, "B", 5, 1000000) + delete(1)805        result = self.run_feed(data, extra_args=("--depth", "5", "--check-book"))806        self.assertEqual(result.returncode, 0, result.stderr)807        rows = list(csv.DictReader(io.StringIO(result.stdout)))808        self.assertEqual((rows[0]["bid_depth"], rows[0]["ask_depth"]), ("100.0000:5", ""))809        self.assertEqual((rows[1]["bid_depth"], rows[1]["ask_depth"]), ("", ""))810811    def test_depth_argument_validation(self):812        for value in ("0", "-1", "1001", "abc", "2x"):813            with self.subTest(value=value):814                result = self.run_feed(directory(), extra_args=("--depth", value))815                self.assertNotEqual(result.returncode, 0)816                self.assertIn("--depth must be", result.stderr)817818    def test_malformed_and_simulated_missing_or_out_of_order(self):819        scenarios = [820            (b"\x00", "truncated frame length"),821            (b"\x00\x24A", "truncated payload"),822            (frame(b"A"), "invalid A length"),823            (frame(b"Z"), "unknown ITCH type"),824            (directory() + reduce("X", 404, 2), "unknown order reference"),  # missing add825            (directory() + reduce("E", 1, 2) + add(1, "B", 10, 1000000), "unknown order reference"),  # reordered826            (directory() + add(1, "B", 10, 1000000) + reduce("X", 1, 11), "invalid share reduction"),827            (directory() + add(1, "B", 10, 1000000) + add(1, "B", 10, 1000000), "duplicate order"),828            (directory() + add(1, "B", 0, 1000000), "invalid add fields"),829            (directory() + add(1, "Q", 10, 1000000), "invalid add fields"),830            (directory() + add(1, "B", 10, 0), "invalid add fields"),831            (directory() + add(1, "B", 10, 1000000, name="OTHER"), "add symbol/locate mismatch"),832            (directory() + add(1, "B", 10, 1000000, locate=8), "add symbol/locate mismatch"),833            (directory() + add(1, "B", 10, 1000000) + replace(1, 1, 5, 999000), "invalid replacement"),834            (directory() + add(1, "B", 10, 1000000) + replace(1, 2, 0, 999000), "invalid replacement"),835        ]836        for data, reason in scenarios:837            with self.subTest(reason=reason):838                result = self.run_feed(data, complete=(reason != "truncated frame length"))839                self.assertNotEqual(result.returncode, 0)840                self.assertIn(reason, result.stderr)841        incomplete = self.run_feed(directory(), complete=False)842        self.assertIn("missing BinaryFILE end marker", incomplete.stderr)843        trailing = self.run_feed(directory() + b"\x00\x00X", complete=False)844        self.assertIn("bytes after BinaryFILE end marker", trailing.stderr)845846847if __name__ == "__main__":848    EXE = Path(sys.argv[1]).resolve()849    sys.argv = [sys.argv[0]]850    unittest.main()