benchmark_prepared_real_window.py
1"""Real-day peak-window performance gate using actual prepared source and financial UOW."""23import argparse4from pathlib import Path5import json6import sys7import tempfile8import time9from unittest.mock import patch1011ROOT = Path(__file__).resolve().parents[1]12sys.path.insert(0, str(ROOT))13from sim_platform import Platform14from sim_platform.experiments.execution.metrics import RuntimeMetrics15from benchmark_prepared_runtime import distribution161718def main():19 parser = argparse.ArgumentParser()20 parser.add_argument('--source', type=Path, required=True)21 parser.add_argument('--output', type=Path, required=True)22 parser.add_argument('--profile-directory', type=Path)23 args = parser.parse_args()24 profile = json.loads((ROOT / 'tools/fixtures/prepared-runtime.json').read_text())25 report = dict(workload='real-source-window', cases=[], passed=False, complete=False)26 with tempfile.TemporaryDirectory(prefix='prepared-real-window-') as temporary:27 workspace = Path(temporary)28 with Platform(workspace) as app:29 source = app._new_research_market('AAPL')30 started = time.monotonic()31 source.load(args.source, allow_implicit_eof=True)32 while not source.prepare():33 time.sleep(0.01)34 report['prepare_seconds'] = time.monotonic() - started35 metadata = source.reader.manifest36 peak = metadata['history']['peak_second_ns']37 report.update(38 source_frames=metadata['source_frames'],39 retained_events=metadata['frames'],40 peak_second_ns=peak,41 peak_events_per_second=metadata['history']['peak_events_per_second'],42 artifact_bytes=sum(p.stat().st_size for p in source.reader.directory.iterdir()),43 scope='three-second real window starting at the highest one-second related-event count; warm local cache',44 )45 from collections import defaultdict4647 regular_density = defaultdict(int)48 for index in range(49 source.reader.prefix(34_200_000_000_000 - 1),50 source.reader.prefix(57_600_000_000_000 - 1),51 ):52 regular_density[source.reader.event_time(index) // 1_000_000_000] += 153 windows = [('all source', peak, metadata['history']['peak_events_per_second'])]54 if regular_density:55 regular = max(regular_density, key=regular_density.get)56 windows.append(57 ('regular trading hours', regular * 1_000_000_000, regular_density[regular])58 )59 report['windows'] = [60 dict(name=name, start_ns=stamp, events_per_second=density)61 for name, stamp, density in windows62 ]63 source.close()64 original = app._compose_pipeline6566 def bounded(driver, market, **options):67 options.update(start_ns=peak, end_ns=peak + 3_000_000_000)68 return original(driver, market, **options)6970 for window, peak, density in windows:71 for trial in range(3):72 user = app.create_profile('real peak ' + str(trial))73 account = app.create_account(user, 'cash', 10_000_000_000)74 with patch.object(app, '_compose_pipeline', bounded):75 experiment = app.open_experiment(76 user,77 account,78 source=args.source,79 method='human',80 settings=dict(81 start_mode='custom', start_time_ns=peak, allow_implicit_eof=True82 ),83 execution_mode='live',84 )85 pipeline = experiment.runner.pipeline86 if args.profile_directory:87 import cProfile8889 args.profile_directory.mkdir(parents=True, exist_ok=True)90 run = pipeline._run9192 def profiled(run=run, trial=trial):93 profiler = cProfile.Profile()94 try:95 profiler.runcall(run)96 finally:97 profiler.dump_stats(98 str(args.profile_directory / ('owner-' + str(trial) + '.stats'))99 )100101 pipeline._run = profiled102 delays = []103 commands = []104 queries = []105106 class Measured(RuntimeMetrics):107 def frame(self, target, event):108 delays.append(max(0, target - event))109 super().frame(target, event)110111 def span(self, target, event, count):112 delays.extend([max(0, target - event)] * count)113 super().span(target, event, count)114115 pipeline.metrics = Measured(100_000_000)116 experiment.play()117 start = time.monotonic()118 next_order = start + 0.1119 next_read = start + 0.05120 number = 0121 while time.monotonic() - start < 3 and experiment.runner.active:122 now = time.monotonic()123 if now >= next_order:124 number += 1125 next_order += 1126 began = time.perf_counter_ns()127 experiment.runner.act(128 [129 dict(130 kind='submit',131 intent=dict(132 client_id='peak-' + str(number),133 symbol='AAPL',134 side='buy',135 quantity=1,136 limit_price=1,137 ),138 )139 ]140 )141 commands.append(time.perf_counter_ns() - began)142 if now >= next_read:143 next_read += 0.1144 began = time.perf_counter_ns()145 pipeline.reader.snapshot_at(pipeline.clock.target_ns())146 queries.append(time.perf_counter_ns() - began)147 time.sleep(0.001)148 until = time.monotonic() + 3149 while experiment.runner.active and time.monotonic() < until:150 time.sleep(0.002)151 state = pipeline.snapshot()152 expected = pipeline.reader.prefix(peak + 3_000_000_000 - 1)153 caught_up = pipeline.driver.sequence == expected154 began = time.monotonic()155 experiment.close()156 shutdown = time.monotonic() - began157 d, c, q = distribution(delays), distribution(commands), distribution(queries)158 thresholds = profile['thresholds']159 passed = (160 state['state'] == 'completed'161 and caught_up162 and d['p99_ms'] <= thresholds['market_processing_delay_p99_ms']163 and c['p99_ms'] <= thresholds['command_roundtrip_p99_ms']164 and q['p99_ms'] <= thresholds['market_query_roundtrip_p99_ms']165 and shutdown <= thresholds['shutdown_seconds_max']166 )167 report['cases'].append(168 dict(169 window=window,170 trial=trial + 1,171 passed=passed,172 state=state['state'],173 caught_up=caught_up,174 market_processing_delay=d,175 financial_command=c,176 market_lookup=q,177 shutdown_seconds=shutdown,178 error=state['error'],179 )180 )181 args.output.write_text(json.dumps(report, indent=2) + '\n')182 print(window, trial + 1, passed, flush=True)183 report['complete'] = True184 report['passed'] = all(c['passed'] for c in report['cases'])185 args.output.write_text(json.dumps(report, indent=2) + '\n')186 print(json.dumps(report, indent=2), flush=True)187188189if __name__ == '__main__':190 main()