m/market labResearch, with a record.
Source on GitHub ↗

benchmark_prepared_real_window.py

1"""Real-day peak-window performance gate using actual prepared source and financial UOW."""23import argparse4from pathlib import Path5import json6import sys7import tempfile8import time9from unittest.mock import patch1011ROOT = Path(__file__).resolve().parents[1]12sys.path.insert(0, str(ROOT))13from sim_platform import Platform14from sim_platform.experiments.execution.metrics import RuntimeMetrics15from benchmark_prepared_runtime import distribution161718def main():19    parser = argparse.ArgumentParser()20    parser.add_argument('--source', type=Path, required=True)21    parser.add_argument('--output', type=Path, required=True)22    parser.add_argument('--profile-directory', type=Path)23    args = parser.parse_args()24    profile = json.loads((ROOT / 'tools/fixtures/prepared-runtime.json').read_text())25    report = dict(workload='real-source-window', cases=[], passed=False, complete=False)26    with tempfile.TemporaryDirectory(prefix='prepared-real-window-') as temporary:27        workspace = Path(temporary)28        with Platform(workspace) as app:29            source = app._new_research_market('AAPL')30            started = time.monotonic()31            source.load(args.source, allow_implicit_eof=True)32            while not source.prepare():33                time.sleep(0.01)34            report['prepare_seconds'] = time.monotonic() - started35            metadata = source.reader.manifest36            peak = metadata['history']['peak_second_ns']37            report.update(38                source_frames=metadata['source_frames'],39                retained_events=metadata['frames'],40                peak_second_ns=peak,41                peak_events_per_second=metadata['history']['peak_events_per_second'],42                artifact_bytes=sum(p.stat().st_size for p in source.reader.directory.iterdir()),43                scope='three-second real window starting at the highest one-second related-event count; warm local cache',44            )45            from collections import defaultdict4647            regular_density = defaultdict(int)48            for index in range(49                source.reader.prefix(34_200_000_000_000 - 1),50                source.reader.prefix(57_600_000_000_000 - 1),51            ):52                regular_density[source.reader.event_time(index) // 1_000_000_000] += 153            windows = [('all source', peak, metadata['history']['peak_events_per_second'])]54            if regular_density:55                regular = max(regular_density, key=regular_density.get)56                windows.append(57                    ('regular trading hours', regular * 1_000_000_000, regular_density[regular])58                )59            report['windows'] = [60                dict(name=name, start_ns=stamp, events_per_second=density)61                for name, stamp, density in windows62            ]63            source.close()64            original = app._compose_pipeline6566            def bounded(driver, market, **options):67                options.update(start_ns=peak, end_ns=peak + 3_000_000_000)68                return original(driver, market, **options)6970            for window, peak, density in windows:71                for trial in range(3):72                    user = app.create_profile('real peak ' + str(trial))73                    account = app.create_account(user, 'cash', 10_000_000_000)74                    with patch.object(app, '_compose_pipeline', bounded):75                        experiment = app.open_experiment(76                            user,77                            account,78                            source=args.source,79                            method='human',80                            settings=dict(81                                start_mode='custom', start_time_ns=peak, allow_implicit_eof=True82                            ),83                            execution_mode='live',84                        )85                    pipeline = experiment.runner.pipeline86                    if args.profile_directory:87                        import cProfile8889                        args.profile_directory.mkdir(parents=True, exist_ok=True)90                        run = pipeline._run9192                        def profiled(run=run, trial=trial):93                            profiler = cProfile.Profile()94                            try:95                                profiler.runcall(run)96                            finally:97                                profiler.dump_stats(98                                    str(args.profile_directory / ('owner-' + str(trial) + '.stats'))99                                )100101                        pipeline._run = profiled102                    delays = []103                    commands = []104                    queries = []105106                    class Measured(RuntimeMetrics):107                        def frame(self, target, event):108                            delays.append(max(0, target - event))109                            super().frame(target, event)110111                        def span(self, target, event, count):112                            delays.extend([max(0, target - event)] * count)113                            super().span(target, event, count)114115                    pipeline.metrics = Measured(100_000_000)116                    experiment.play()117                    start = time.monotonic()118                    next_order = start + 0.1119                    next_read = start + 0.05120                    number = 0121                    while time.monotonic() - start < 3 and experiment.runner.active:122                        now = time.monotonic()123                        if now >= next_order:124                            number += 1125                            next_order += 1126                            began = time.perf_counter_ns()127                            experiment.runner.act(128                                [129                                    dict(130                                        kind='submit',131                                        intent=dict(132                                            client_id='peak-' + str(number),133                                            symbol='AAPL',134                                            side='buy',135                                            quantity=1,136                                            limit_price=1,137                                        ),138                                    )139                                ]140                            )141                            commands.append(time.perf_counter_ns() - began)142                        if now >= next_read:143                            next_read += 0.1144                            began = time.perf_counter_ns()145                            pipeline.reader.snapshot_at(pipeline.clock.target_ns())146                            queries.append(time.perf_counter_ns() - began)147                        time.sleep(0.001)148                    until = time.monotonic() + 3149                    while experiment.runner.active and time.monotonic() < until:150                        time.sleep(0.002)151                    state = pipeline.snapshot()152                    expected = pipeline.reader.prefix(peak + 3_000_000_000 - 1)153                    caught_up = pipeline.driver.sequence == expected154                    began = time.monotonic()155                    experiment.close()156                    shutdown = time.monotonic() - began157                    d, c, q = distribution(delays), distribution(commands), distribution(queries)158                    thresholds = profile['thresholds']159                    passed = (160                        state['state'] == 'completed'161                        and caught_up162                        and d['p99_ms'] <= thresholds['market_processing_delay_p99_ms']163                        and c['p99_ms'] <= thresholds['command_roundtrip_p99_ms']164                        and q['p99_ms'] <= thresholds['market_query_roundtrip_p99_ms']165                        and shutdown <= thresholds['shutdown_seconds_max']166                    )167                    report['cases'].append(168                        dict(169                            window=window,170                            trial=trial + 1,171                            passed=passed,172                            state=state['state'],173                            caught_up=caught_up,174                            market_processing_delay=d,175                            financial_command=c,176                            market_lookup=q,177                            shutdown_seconds=shutdown,178                            error=state['error'],179                        )180                    )181                    args.output.write_text(json.dumps(report, indent=2) + '\n')182                    print(window, trial + 1, passed, flush=True)183    report['complete'] = True184    report['passed'] = all(c['passed'] for c in report['cases'])185    args.output.write_text(json.dumps(report, indent=2) + '\n')186    print(json.dumps(report, indent=2), flush=True)187188189if __name__ == '__main__':190    main()