benchmark_prepared_runtime.py
1"""Prepared-runtime baseline and Agent integration performance gate; actual Platform/Runtime/financial UOW.23Uses a fixed predeclared profile, bounded synthetic source, mock/local IO only.4Deployment evidence combines these results with real-source and regression gates.5"""67from pathlib import Path8import argparse9import asyncio10import multiprocessing11import json12import statistics13import sys14import tempfile15import time16from unittest.mock import patch1718ROOT = Path(__file__).resolve().parents[1]19sys.path.insert(0, str(ROOT))20sys.path.insert(0, str(ROOT / 'tests'))21sys.path.insert(0, str(ROOT / 'tests' / 'platform'))22from itch_fixtures import directory, add23from sim_platform import Platform242526def distribution(values):27 if not values:28 return dict(count=0, median_ms=None, p99_ms=None, max_ms=None)29 values = sorted(values)30 return dict(31 count=len(values),32 median_ms=statistics.median(values) / 1e6,33 p99_ms=values[min(len(values) - 1, int(0.99 * len(values)))] / 1e6,34 max_ms=max(values) / 1e6,35 )363738def fixture(path, profile):39 period = 1_000_000_000 // profile['source_events_per_second']40 count = profile['case_seconds'] * profile['source_events_per_second']41 with path.open('wb') as output:42 output.write(directory('AAPL', timestamp=1))43 output.write(add(1, 'B', 100, 99, 'AAPL', timestamp=2))44 output.write(add(2, 'S', 100, 101, 'AAPL', timestamp=3))45 for index in range(count):46 output.write(add(3 + index, 'S', 1, 101, 'AAPL', timestamp=3 + (index + 1) * period))47 output.write(b'\0\0')48 return path495051def run_case(workspace, source, profile, case, trial):52 from sim_platform.experiments.execution.metrics import RuntimeMetrics5354 with Platform(workspace) as app:55 user = app.create_profile('runtime gate')56 account = app.create_account(user, 'cash', 100_000_000)57 experiment = app.open_experiment(user, account, source=source, execution_mode='live')58 pipeline = experiment.runner.pipeline59 frames, commands, queue_wait = [], [], []6061 class Measured(RuntimeMetrics):62 def frame(self, target, event):63 frames.append(max(0, target - event))64 super().frame(target, event)6566 def span(self, target, event, count):67 frames.extend([max(0, target - event)] * count)68 super().span(target, event, count)6970 pipeline.metrics = Measured(100_000_000)71 original = pipeline.operation7273 def timed(request, position):74 queue_wait.append(max(0, pipeline.clock.target_ns() - int(position['time_ns'])))75 return original(request, position)7677 pipeline.operation = timed78 queries = []79 experiment.play()80 start = time.monotonic()81 next_order = start + 0.182 next_read = start + 0.0583 deadline = start + profile['case_seconds']84 number = 085 while time.monotonic() < deadline and experiment.runner.active:86 now = time.monotonic()87 if case != 'idle' and now >= next_order:88 number += 189 next_order += 1 / profile['orders_per_second']90 begun = time.perf_counter_ns()91 experiment.runner.act(92 [93 dict(94 kind='submit',95 intent=dict(96 client_id='gate-' + str(number),97 symbol='AAPL',98 side='buy',99 quantity=1,100 limit_price=101 if case == 'fills_and_reads' else 1,101 ),102 )103 ]104 )105 commands.append(time.perf_counter_ns() - begun)106 if case != 'idle' and now >= next_read:107 next_read += 1 / profile['market_queries_per_second']108 begun = time.perf_counter_ns()109 cutoff = min(pipeline.clock.target_ns(), pipeline.reader.manifest['last_ns'])110 pipeline.reader.snapshot_at(cutoff)111 queries.append(time.perf_counter_ns() - begun)112 time.sleep(0.001)113 until = time.monotonic() + 2114 while experiment.runner.active and time.monotonic() < until:115 time.sleep(0.002)116 state = pipeline.snapshot()117 begun = time.monotonic()118 experiment.close()119 shutdown = time.monotonic() - begun120 thresholds = profile['thresholds']121 delay = distribution(frames)122 command = distribution(commands)123 query = distribution(queries)124 complete = pipeline.driver.sequence == pipeline.reader.manifest['frames']125 passed = (126 complete127 and state['state'] == 'completed'128 and delay['p99_ms'] <= thresholds['market_processing_delay_p99_ms']129 and (not commands or command['p99_ms'] <= thresholds['command_roundtrip_p99_ms'])130 and (not queries or query['p99_ms'] <= thresholds['market_query_roundtrip_p99_ms'])131 and shutdown <= thresholds['shutdown_seconds_max']132 )133 return dict(134 case=case,135 trial=trial,136 passed=passed,137 completed=complete,138 state=state['state'],139 financial_commands=command,140 market_lookup=query,141 owner_queue_wait=distribution(queue_wait),142 market_processing_delay=delay,143 shutdown_seconds=shutdown,144 query_scope='trusted indexed lookup, not complete Tools IPC; Agent tool roundtrip is a separate gate',145 )146147148class ParsingProvider:149 async def request_received(self, profile, payload, secret, received):150 await asyncio.sleep(0.02)151 received()152 # CPU parsing/load is isolated in Agent, not sleeping on the App GIL.153 deadline = time.monotonic() + 0.2154 value = 1155 while time.monotonic() < deadline:156 value = (value * 1664525 + 1013904223) % 2**32157 return dict(output=[], sample='x' * 50000)158159160class ModelWorkload:161 def __init__(self, finished):162 self.finished = finished163164 def __call__(self, client, configuration):165 work = client.next_work()166 result = client.request_model(work, {'input': 'local performance fixture'})167 if result['output'] != []:168 raise RuntimeError('unexpected fixture model result')169 self.finished.value = 1170 client.wait_stopped()171172173def agent_case(workspace, source, profile, trial):174 from functools import partial175 from support.agents import definition, settings176 from support.tools import request177 from sim_platform.agents.hosting.host import AgentProcessHost178 from sim_platform.models.contracts import ModelProfile179 import threading180181 finished = multiprocessing.get_context('spawn').RawValue('b', 0)182 with Platform(workspace) as app:183 user = app.create_profile('Agent gate')184 account = app.create_account(user, 'cash', 100_000_000)185 app.create_agent(user, definition())186 config = dict(187 settings(),188 start_ns='3',189 end_ns=str(3 + profile['case_seconds'] * 1_000_000_000),190 timeout_seconds='2',191 granted_tools=['market.quote', 'orders.submit_limit'],192 )193 with patch(194 'sim_platform.agents.hosting.host.AgentProcessHost',195 partial(AgentProcessHost, program=ModelWorkload(finished)),196 ):197 experiment = app.open_experiment(198 user,199 account,200 source=source,201 method='agent',202 agent_id='demo',203 revision=1,204 settings=config,205 execution_mode='live',206 )207 app.set_model_credential('gate', 'local-fixture-not-an-api-key')208 app.configure_experiment_model(209 experiment,210 ModelProfile('openai-responses', 'mock', 'gate'),211 provider=ParsingProvider(),212 )213 pipeline = experiment.runner.pipeline214 from sim_platform.experiments.execution.metrics import RuntimeMetrics215216 delays = []217218 class Measured(RuntimeMetrics):219 def frame(self, target, event):220 delays.append(max(0, target - event))221 super().frame(target, event)222223 def span(self, target, event, count):224 delays.extend([max(0, target - event)] * count)225 super().span(target, event, count)226227 pipeline.metrics = Measured(100_000_000)228 queries = []229 commands = []230 number = 0231 experiment.play()232 start = time.monotonic()233 next_read = start + 0.05234 next_order = start + 0.1235 while time.monotonic() - start < profile['case_seconds'] and experiment.runner.active:236 work = experiment.runner._tools.controls.snapshot()237 if work['state'] != 'running':238 time.sleep(0.001)239 continue240 now = time.monotonic()241 if now >= next_read:242 number += 1243 next_read += 1 / profile['market_queries_per_second']244 value = request(245 'market.quote',246 {'symbol': 'AAPL'},247 number='read-' + str(number),248 run=experiment.run_id,249 )250 value.update(work_id=work['work_id'], epoch=work['epoch'])251 begun = time.perf_counter_ns()252 response = experiment.request_tool(value).result(2)['result']253 if response['status'] != 'ok':254 raise RuntimeError('query failed')255 queries.append(time.perf_counter_ns() - begun)256 if now >= next_order:257 number += 1258 next_order += 1 / profile['orders_per_second']259 value = request(260 'orders.submit_limit',261 dict(symbol='AAPL', side='buy', quantity_shares=1, limit_price_usd='0.0001'),262 number='order-' + str(number),263 run=experiment.run_id,264 )265 value.update(work_id=work['work_id'], epoch=work['epoch'])266 begun = time.perf_counter_ns()267 response = experiment.request_tool(value).result(2)['result']268 if response['status'] != 'ok':269 raise RuntimeError('order failed')270 commands.append(time.perf_counter_ns() - begun)271 time.sleep(0.001)272 until = time.monotonic() + 2273 while experiment.runner.active and time.monotonic() < until:274 time.sleep(0.002)275 state = pipeline.snapshot()276 begun = time.monotonic()277 experiment.close()278 shutdown = time.monotonic() - begun279 q = distribution(queries)280 c = distribution(commands)281 d = distribution(delays)282 threshold = profile['thresholds']283 no_model_thread = not any(t.name == 'model-async' for t in threading.enumerate())284 passed = (285 bool(finished.value)286 and state['state'] == 'completed'287 and no_model_thread288 and len(queries) > 0289 and q['p99_ms'] <= threshold['market_query_roundtrip_p99_ms']290 and len(commands) > 0291 and c['p99_ms'] <= threshold['command_roundtrip_p99_ms']292 and d['p99_ms'] <= threshold['market_processing_delay_p99_ms']293 and shutdown <= threshold['shutdown_seconds_max']294 )295 return dict(296 case='Agent model IO/CPU parsing + complete Tools IPC',297 trial=trial,298 passed=passed,299 model_completed=bool(finished.value),300 no_app_model_thread=no_model_thread,301 market_query_roundtrip=q,302 financial_tool_roundtrip=c,303 market_processing_delay=d,304 state=state['state'],305 shutdown_seconds=shutdown,306 )307308309def main():310 parser = argparse.ArgumentParser()311 parser.add_argument('--output', type=Path, required=True)312 parser.add_argument(313 '--include-agent', action='store_true', help='Also measure Agent model IO and tool load.'314 )315 args = parser.parse_args()316 profile = json.loads((ROOT / 'tools/fixtures/prepared-runtime.json').read_text())317 report = dict(318 include_agent=args.include_agent,319 profile=profile,320 deployment='App dedicated execution thread',321 cases=[],322 complete=False,323 passed=False,324 )325 with tempfile.TemporaryDirectory(prefix='prepared-runtime-gate-') as directory:326 root = Path(directory)327 source = fixture(root / 'source.bin', profile)328 for case in profile['cases']:329 for trial in range(profile['repetitions']):330 result = run_case(root / 'app', source, profile, case, trial + 1)331 report['cases'].append(result)332 args.output.write_text(json.dumps(report, indent=2) + '\n')333 print(case, trial + 1, result['passed'], flush=True)334 if args.include_agent:335 report['agent_cases'] = []336 for trial in range(profile['repetitions']):337 result = agent_case(root / ('agent-' + str(trial + 1)), source, profile, trial + 1)338 report['agent_cases'].append(result)339 args.output.write_text(json.dumps(report, indent=2) + '\n')340 print('Agent model/Tools', trial + 1, result['passed'], flush=True)341 report['complete'] = True342 report['passed'] = all(case['passed'] for case in report['cases']) and all(343 case['passed'] for case in report.get('agent_cases', [])344 )345 args.output.write_text(json.dumps(report, indent=2) + '\n')346347348if __name__ == '__main__':349 main()