benchmark_platform.py
1"""Measure the current platform without changing its execution semantics.23Each trial runs in a fresh child process and uses a temporary account database.4The real-data fixture is a validated PREFIX, with an artificial end marker:5it retains all preceding frames so that the native book can be reconstructed.6Preparation is excluded from playback timings. Qt trials use the actual worker,7queued commands and window rendering, with the offscreen platform.8"""910import argparse11from collections import Counter, defaultdict12from contextlib import contextmanager13from dataclasses import asdict14from functools import wraps15import hashlib16import json17import os18from pathlib import Path19import platform20import random2122try:23 import resource # POSIX only; peak memory is not reported on Windows.24except ImportError:25 resource = None # type: ignore[assignment]26import statistics27import struct28import subprocess29import sys30from tempfile import TemporaryDirectory31import threading32import time3334ROOT = Path(__file__).resolve().parents[1]35sys.path.insert(0, str(ROOT))36CASES = (37 'native',38 'market',39 'execution_idle',40 'human_idle',41 'human_trade',42 'rule_idle',43 'rule_trade',44 'desktop_idle',45 'desktop_trade',46)474849def sha256(path):50 result = hashlib.sha256()51 with Path(path).open('rb') as stream:52 for chunk in iter(lambda: stream.read(4 * 1024 * 1024), b''):53 result.update(chunk)54 return result.hexdigest()555657def save(path, value):58 path = Path(path)59 temporary = path.with_suffix(path.suffix + '.tmp')60 temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n')61 temporary.replace(path)626364def percentile(values, fraction):65 if not values:66 return None67 ordered = sorted(values)68 return ordered[max(0, min(len(ordered) - 1, int(len(ordered) * fraction + 0.999999) - 1))]697071def prepare_fixture(source, destination, seconds, symbol, start_ns=None):72 """C++ validates/applies frames; Python only copies their framing verbatim."""73 from sim_platform.market.native import NativeEngine7475 before = source.stat()76 engine = NativeEngine(symbol)77 engine.open_file(source, True)78 origin = None79 book = None80 copied = 081 previous_ns = None82 started = time.perf_counter()83 try:84 with source.open('rb') as stream, destination.open('wb') as output:85 while True:86 candidate = engine.peek_file()87 if candidate is None:88 break89 stamp = candidate['timestamp_ns']90 if previous_ns is not None and stamp < previous_ns:91 raise ValueError('source time moved backwards')92 if origin and stamp > origin['time_ns'] + int(seconds * 1e9):93 break94 result = engine.next_file()95 prefix = stream.read(2)96 if len(prefix) != 2:97 raise ValueError('copy stream ended before native reader')98 length = int.from_bytes(prefix, 'big')99 payload = stream.read(length)100 if len(payload) != length or not length:101 raise ValueError('copy stream framing differs from native reader')102 output.write(prefix + payload)103 copied += 1104 previous_ns = stamp105 if result['book']:106 book = result['book']107 if (108 origin is None109 and book110 and book['bids']111 and book['asks']112 and (start_ns is None or stamp >= start_ns)113 ):114 origin = dict(sequence=copied, time_ns=stamp)115 if copied % 2_000_000 == 0:116 print(f'Preparing prefix: {copied:,} frames', flush=True)117 output.write(b'\0\0')118 finally:119 engine.close()120 after = source.stat()121 if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns):122 raise ValueError('source changed while preparing fixture')123 if not origin or copied <= origin['sequence']:124 raise ValueError('source has no measurable interval after a two-sided book')125 return dict(126 kind='real-data-prefix',127 origin=origin,128 final_sequence=copied,129 end_time_ns=previous_ns,130 playback_frames=copied - origin['sequence'],131 artificial_end_marker=True,132 requested_seconds=seconds,133 preparation_seconds=time.perf_counter() - started,134 sha256=sha256(destination),135 bytes=destination.stat().st_size,136 )137138139def synthetic_fixture(path, frames, seconds):140 """Constant-size book: directory, bid/ask, then repeated replacement messages."""141 stock = b'AAPL '142 start = 34_200_000_000_000143144 def header(kind, stamp):145 return kind + struct.pack('>HH', 7, 0) + stamp.to_bytes(6, 'big')146147 def write(stream, payload):148 stream.write(struct.pack('>H', len(payload)) + payload)149150 with path.open('wb') as stream:151 write(152 stream,153 header(b'R', start)154 + stock155 + b'QN'156 + struct.pack('>I', 100)157 + b'NC PNN1N'158 + struct.pack('>I', 1)159 + b'N',160 )161 for order, side, price in ((1, b'B', 1_900_000), (2, b'S', 1_901_000)):162 write(163 stream,164 header(b'A', start + order)165 + struct.pack('>Q', order)166 + side167 + struct.pack('>I', 100)168 + stock169 + struct.pack('>I', price),170 )171 previous = 2172 for index in range(1, frames + 1):173 stamp = start + 2 + int(index * seconds * 1e9 / frames)174 current = index + 2175 write(176 stream,177 header(b'U', stamp) + struct.pack('>QQII', previous, current, 100, 1_901_000),178 )179 previous = current180 stream.write(b'\0\0')181 return dict(182 kind='synthetic-constant-book',183 origin=dict(sequence=3, time_ns=start + 2),184 final_sequence=frames + 3,185 end_time_ns=start + 2 + int(seconds * 1e9),186 playback_frames=frames,187 artificial_end_marker=True,188 requested_seconds=seconds,189 sha256=sha256(path),190 bytes=path.stat().st_size,191 )192193194class Timings:195 """Opt-in, trial-local instrumentation; exclusive totals avoid double counting."""196197 def __init__(self):198 self.enabled = False199 self.local = threading.local()200 self.totals = defaultdict(lambda: [0, 0, 0])201 self.patches = []202203 @contextmanager204 def span(self, name):205 if not self.enabled:206 yield207 return208 stack = getattr(self.local, 'stack', None)209 if stack is None:210 stack = self.local.stack = []211 item = [time.perf_counter_ns(), 0]212 stack.append(item)213 try:214 yield215 finally:216 elapsed = time.perf_counter_ns() - item[0]217 stack.pop()218 if stack:219 stack[-1][1] += elapsed220 total = self.totals[name]221 total[0] += 1222 total[1] += elapsed223 total[2] += elapsed - item[1]224225 def wrap(self, owner, attribute, name):226 original = getattr(owner, attribute)227228 @wraps(original)229 def measured(*args, **kwargs):230 with self.span(name):231 return original(*args, **kwargs)232233 self.patches.append((owner, attribute, original))234 setattr(owner, attribute, measured)235236 def install(self, desktop=False):237 from sim_platform.market.native import NativeEngine238 from sim_platform.market.session import MarketSession239 from sim_platform.execution.service import ExecutionService240 from sim_platform.decisions.policy.runner import PolicyDecisionRunner241 from sim_platform.decisions.policy import invocation242 from sim_platform.persistence.sql.database import Database243 from sim_platform.research.controller import ResearchController244 from sim_platform.persistence.workspace.market_cache import MarketCacheFiles245 from sqlalchemy.pool import Pool246 from sqlalchemy.engine import Connection247248 for method in ('peek_file', 'next_file'):249 self.wrap(NativeEngine, method, 'bridge.' + method)250 self.wrap(NativeEngine, '_decode', 'bridge.json_decode')251 self.wrap(MarketSession, '_advance', 'market.advance')252 self.wrap(ExecutionService, 'advance', 'execution.advance')253 self.wrap(PolicyDecisionRunner, 'observe', 'decision.observe')254 self.wrap(PolicyDecisionRunner, '_decide', 'decision.complete')255 self.wrap(invocation, 'invoke', 'decision.invoke_validate')256 self.wrap(ResearchController, 'view', 'presentation.view')257 self.wrap(Pool, 'connect', 'database.acquire_connection')258 self.wrap(Connection, '_commit_impl', 'database.commit')259 self.wrap(Connection, '_exec_single_context', 'database.sql_execute')260 for method in ('append', 'read', 'write'):261 self.wrap(MarketCacheFiles, method, 'history.' + method)262 original = Database.unit_of_work263264 @contextmanager265 def measured(database, *, write=False):266 with self.span('database.write_uow' if write else 'database.read_uow'):267 with original(database, write=write) as session:268 yield session269270 self.patches.append((Database, 'unit_of_work', original))271 Database.unit_of_work = measured272 if desktop:273 from sim_platform.desktop.web.window import WebResearchWindow274275 self.wrap(WebResearchWindow, 'publish', 'presentation.qt_render')276277 def close(self):278 self.enabled = False279 for owner, attribute, original in reversed(self.patches):280 setattr(owner, attribute, original)281282 def result(self):283 return {284 name: dict(285 calls=value[0], inclusive_seconds=value[1] / 1e9, exclusive_seconds=value[2] / 1e9286 )287 for name, value in sorted(self.totals.items())288 }289290291class IdlePolicy:292 kind, name, version = 'rule', 'benchmark-no-actions', '1'293 requires = ('book', 'account')294 parameters = {}295296 def decide(self, observation, state, timeout_ms):297 return dict(actions=[], state=state)298299300def canonical(market, account=None):301 value = {key: market[key] for key in ('sequence', 'time_ns', 'bids', 'asks')}302 if account is not None:303 value['account'] = {304 key: account[key] for key in ('cash', 'fees', 'realized_pnl', 'positions')305 }306 value['account']['orders'] = [307 {key: order[key] for key in ('side', 'quantity', 'limit_price', 'filled', 'status')}308 for order in account['orders']309 ]310 encoded = json.dumps(value, sort_keys=True, separators=(',', ':'))311 return dict(data=value, sha256=hashlib.sha256(encoded.encode()).hexdigest())312313314class Progress:315 def __init__(self, origin):316 self.origin = origin317 self.bins = Counter()318 self.boundaries = []319 self.last_second = 0320 self.started = time.perf_counter()321322 def record(self, stamp, count=1):323 second = max(0, (stamp - self.origin['time_ns']) // 1_000_000_000)324 self.bins[second] += count325 if second > self.last_second:326 now = time.perf_counter() - self.started327 self.boundaries.append(dict(simulated_second=second, elapsed_seconds=now))328 self.last_second = second329330331def core_trial(args, directory, timings):332 from sim_platform import Platform333 from sim_platform.market.native import NativeEngine334 from sim_platform.decisions import DecisionConfig335 from sim_platform.execution import ExecutionConfig336337 meta = json.loads(args.fixture_meta.read_text())338 origin = meta['origin']339 started = time.perf_counter()340 owner = None341 platform_root = None342 controller = None343 native = None344 preparation = None345 setup = None346 try:347 if args.worker == 'native':348 native = NativeEngine(args.symbol)349 native.open_file(args.input)350 book = None351 result = None352 for _ in range(origin['sequence']):353 result = native.next_file()354 book = result['book'] or book355 preparation = time.perf_counter() - started356 setup = 0357 progress = Progress(origin)358 timings.enabled = args.profile359 while True:360 if native.peek_file() is None:361 break362 result = native.next_file()363 book = result['book'] or book364 progress.record(result['timestamp_ns'])365 wall = time.perf_counter() - progress.started366 timings.enabled = False367 market = dict(368 sequence=result['sequence'],369 time_ns=result['timestamp_ns'],370 bids=book['bids'],371 asks=book['asks'],372 )373 status = 'completed'374 account = None375 decisions = 0376 else:377 platform_root = Platform(directory)378 controller = platform_root.open_research()379 controller.begin_prepare(380 args.input, symbol=args.symbol, start_mode='custom', start_time_ns=origin['time_ns']381 )382 while controller.preparation['status'] == 'preparing':383 controller.prepare_tick(max_events=5000, budget_seconds=0.035)384 if controller.market.snapshot.sequence != origin['sequence']:385 raise ValueError('prepared origin differs from fixture manifest')386 preparation = time.perf_counter() - started387 bind_started = time.perf_counter()388 if args.worker != 'market':389 profile = platform_root.create_profile('Benchmark')390 account_id = platform_root.create_account(profile, 'Temporary', 100_000_000)391 if args.worker == 'execution_idle':392 maker = platform_root.create_maker(profile, 'Human', 'human', '1')393 run = platform_root.create_run(394 profile,395 account_id,396 maker,397 [args.symbol],398 dict(399 execution=asdict(ExecutionConfig()), research_source=controller.source400 ),401 )402 owner = platform_root.execution_session(profile, run, maker, controller.market)403 elif args.worker == 'rule_idle':404 policy = IdlePolicy()405 maker, run = platform_root.create_managed_decision_run(406 profile,407 account_id,408 args.symbol,409 policy,410 DecisionConfig(),411 ExecutionConfig(),412 source=controller.source,413 )414 controller.profile, controller.run_id = profile, run415 controller.runner = platform_root.decision_runner(416 profile, run, maker, controller.market, policy417 )418 owner = controller.runner419 else:420 controller.start(421 profile,422 account_id,423 policy='rule' if args.worker == 'rule_trade' else 'human',424 execution_mode='controlled',425 )426 owner = controller.runner427 run = controller.run_id428 if args.worker == 'human_trade':429 controller.order(430 'benchmark-buy', 'buy', 10, controller.market.snapshot.asks[0][0]431 )432 else:433 owner = controller.market434 setup = time.perf_counter() - bind_started435 owner.play()436 progress = Progress(origin)437 timings.enabled = args.profile438 while controller.market.status == 'playing' and not getattr(owner, 'closed', False):439 events = owner.pump(1)440 if not events:441 break442 event = events[0]443 if event.kind in ('market', 'clock'):444 progress.record(event.time_ns)445 if event.snapshot.status == 'failed':446 raise ValueError(event.message or 'market failed')447 wall = time.perf_counter() - progress.started448 timings.enabled = False449 market = asdict(controller.market.snapshot)450 account = (451 asdict(platform_root.snapshot(profile, account_id))452 if args.worker != 'market'453 else None454 )455 status = (456 platform_root.run(profile, run, include_records=False)['status']457 if args.worker != 'market'458 else market['status']459 )460 decisions = (461 len(platform_root.decision_records(profile, run)) if controller.runner else 0462 )463 if status != 'completed':464 raise ValueError('trial did not complete: ' + status)465 return dict(466 wall_seconds=wall,467 preparation_seconds=preparation,468 setup_seconds=setup,469 origin=origin,470 final_sequence=market['sequence'],471 end_time_ns=market['time_ns'],472 frames=market['sequence'] - origin['sequence'],473 decisions=decisions,474 final=canonical(market, account),475 status=status,476 source_second_density=dict(progress.bins),477 boundary_timings=progress.boundaries,478 )479 finally:480 timings.enabled = False481 close_started = time.perf_counter()482 if owner is not None and args.worker == 'execution_idle':483 owner.close()484 if platform_root is not None:485 platform_root.close()486 if native is not None:487 native.close()488 timings.cleanup_seconds = time.perf_counter() - close_started489490491def desktop_trial(args, directory, timings):492 os.environ.setdefault('QT_QPA_PLATFORM', 'offscreen')493 from PySide6.QtCore import QTimer494 from tools.desktop.driver import DesktopDriver495496 meta = json.loads(args.fixture_meta.read_text())497 origin = meta['origin']498 driver = DesktopDriver(directory, (args.input, None), timeout=args.timeout)499 window = driver.window500 until = driver.until501 setup_started = time.perf_counter()502 try:503 driver.prepare(504 args.input,505 policy='rule' if args.worker == 'desktop_trade' else 'human',506 start_ns=origin['time_ns'],507 symbol=args.symbol,508 )509 preparation = time.perf_counter() - setup_started510 setup = 0.0511 if driver.view['market']['sequence'] != origin['sequence']:512 raise ValueError('desktop prepared origin differs')513 frame_times = []514 heartbeat_delays = []515 state = dict(last_heartbeat=time.perf_counter(), previous_sequence=origin['sequence'])516517 def heartbeat():518 now = time.perf_counter()519 heartbeat_delays.append(max(0, now - state['last_heartbeat'] - 0.01))520 state['last_heartbeat'] = now521522 timer = QTimer()523 timer.setInterval(10)524 timer.timeout.connect(heartbeat)525526 def published(view):527 if view.get('market') and view['market']['sequence'] > state['previous_sequence']:528 frame_times.append(time.perf_counter())529 state['previous_sequence'] = view['market']['sequence']530531 window.owners[driver.identity].event.connect(532 lambda kind, value: published(value['value']) if kind == 'view' else None533 )534 timer.start()535 timings.enabled = args.profile536 started = time.perf_counter()537 driver.operation('play', speed=None)538 until(lambda: driver.view['execution']['status'] != 'running')539 wall = time.perf_counter() - started540 timings.enabled = False541 timer.stop()542 view = driver.view543 if view['execution']['status'] != 'completed':544 raise ValueError('desktop run did not complete')545 intervals = [b - a for a, b in zip(frame_times, frame_times[1:])]546 return dict(547 wall_seconds=wall,548 preparation_seconds=preparation,549 setup_seconds=setup,550 origin=origin,551 final_sequence=view['market']['sequence'],552 end_time_ns=view['market']['time_ns'],553 frames=view['market']['sequence'] - origin['sequence'],554 decisions=len(view['decisions']),555 decisions_count_is_tail=True,556 final=canonical(view['market'], view['account']),557 status='completed',558 qt=dict(559 platform='offscreen',560 updates=len(frame_times),561 update_interval_p50_seconds=percentile(intervals, 0.5),562 update_interval_p95_seconds=percentile(intervals, 0.95),563 heartbeat_lateness_p95_seconds=percentile(heartbeat_delays, 0.95),564 ),565 )566 finally:567 timings.enabled = False568 close_started = time.perf_counter()569 driver.close()570 timings.cleanup_seconds = time.perf_counter() - close_started571572573def worker(args):574 timings = Timings()575 started = time.perf_counter()576 result = dict(case=args.worker, instrumented=args.profile)577 try:578 if args.profile:579 timings.install(desktop=args.worker.startswith('desktop'))580 with TemporaryDirectory(prefix='platform-benchmark-') as temporary:581 directory = Path(temporary) / 'accounts'582 function = desktop_trial if args.worker.startswith('desktop') else core_trial583 result.update(function(args, directory, timings))584 result['database_bytes_after_close'] = (585 sum(p.stat().st_size for p in directory.rglob('*') if p.is_file())586 if directory.exists()587 else 0588 )589 seconds = (result['end_time_ns'] - result['origin']['time_ns']) / 1e9590 result.update(591 simulated_seconds=seconds,592 speed_x=seconds / result['wall_seconds'],593 frames_per_second=result['frames'] / result['wall_seconds'],594 cleanup_seconds=timings.cleanup_seconds,595 )596 except Exception as exc:597 result.update(status='failed', error=f'{type(exc).__name__}: {exc}')598 import traceback599600 result['traceback'] = traceback.format_exc()601 finally:602 timings.close()603 rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss if resource else None604 result.update(605 process_seconds=time.perf_counter() - started,606 peak_rss_bytes=(None if rss is None else rss if sys.platform == 'darwin' else rss * 1024),607 timings=timings.result(),608 )609 save(args.result, result)610 return 0 if result['status'] == 'completed' else 1611612613def aggregate(samples):614 rows = []615 for case in CASES:616 trials = [s for s in samples if s['case'] == case and not s['instrumented']]617 if not trials:618 continue619 valid = [s for s in trials if s['status'] == 'completed']620 row = dict(case=case, completed=len(valid), failed=len(trials) - len(valid))621 if valid:622 for key in (623 'wall_seconds',624 'preparation_seconds',625 'speed_x',626 'frames_per_second',627 'peak_rss_bytes',628 'database_bytes_after_close',629 'cleanup_seconds',630 ):631 row[key] = statistics.median(s[key] for s in valid)632 row.update(633 min_wall_seconds=min(s['wall_seconds'] for s in valid),634 max_wall_seconds=max(s['wall_seconds'] for s in valid),635 deterministic=len({s['final']['sha256'] for s in valid}) == 1,636 frames=valid[0]['frames'],637 simulated_seconds=valid[0]['simulated_seconds'],638 )639 rows.append(row)640 return rows641642643def markdown(report):644 rows = [645 '# Current platform performance baseline',646 '',647 'Unpaced throughput; 1× pacing is not implemented by this benchmark. Preparation is excluded.',648 'Fresh subprocess/database for every trial; file cache is uncontrolled (not a cold-cache benchmark).',649 'Qt uses actual window/worker/queued signals with offscreen rendering; physical display latency is unmeasured.',650 '',651 '| Case | Trials | Frames | Simulated s | Playback s (median) | Speed | Frames/s | Peak RSS MiB | DB MiB |',652 '| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |',653 ]654 for row in report['summary']:655 if row['completed']:656 rows.append(657 f"| {row['case']} | {row['completed']} | {row['frames']:,} | {row['simulated_seconds']:.3f} | "658 f"{row['wall_seconds']:.3f} | {row['speed_x']:.3f}× | {row['frames_per_second']:.0f} | "659 f"{row['peak_rss_bytes']/2**20:.1f} | {row['database_bytes_after_close']/2**20:.2f} |"660 )661 else:662 rows.append(f"| {row['case']} | FAILED | | | | | | | |")663 rows += [664 '',665 '## Diagnostic instrumentation',666 '',667 'Separate instrumented trials, excluded from the throughput medians. Exclusive times do not double count nested calls.',668 '',669 ]670 for sample in report['samples']:671 if not sample['instrumented'] or sample['status'] != 'completed':672 continue673 rows += [674 f"### {sample['case']}",675 '',676 '| Component | Calls | Inclusive s | Exclusive s |',677 '| --- | ---: | ---: | ---: |',678 ]679 for name, value in sorted(680 sample['timings'].items(), key=lambda item: -item[1]['exclusive_seconds']681 ):682 rows.append(683 f"| {name} | {value['calls']:,} | {value['inclusive_seconds']:.3f} | {value['exclusive_seconds']:.3f} |"684 )685 rows.append('')686 rows += [687 '## Scope and checks',688 '',689 '- Source prefix includes warm-up frames and an artificial terminator; this is not a full-day result.',690 '- End-of-prefix closes remaining orders normally. Trade outcomes are model/fixture results, not market recommendations.',691 '- RSS includes initialization/preparation/final result collection; it is not steady-state book memory.',692 '- Native includes C++ calls and JSON decoding. It is not C++-only throughput.',693 '- UI heartbeat measures main-thread scheduling delay; it is not queued Pause/Stop acknowledgement.',694 '- Stop/cancel response and full-day scalability require additional probes.',695 f"- Same-case deterministic results: {all(r.get('deterministic', False) for r in report['summary'])}.",696 f"- Cross-layer final-state checks: `{json.dumps(report.get('checks', {}), sort_keys=True)}`.",697 '',698 ]699 return '\n'.join(rows)700701702def main(args):703 args.output = args.output.resolve()704 args.output.mkdir(parents=True, exist_ok=False)705 fixture = args.output / 'prefix.bin'706 preparation_started = time.perf_counter()707 source = args.input.resolve() if args.input else None708 if source:709 source_before = source.stat()710 meta = prepare_fixture(source, fixture, args.seconds, args.symbol, args.start_time_ns)711 hash_started = time.perf_counter()712 identity = dict(713 path=str(source),714 bytes=source_before.st_size,715 sha256=sha256(source),716 hash_seconds=time.perf_counter() - hash_started,717 )718 source_after = source.stat()719 if (source_before.st_size, source_before.st_mtime_ns) != (720 source_after.st_size,721 source_after.st_mtime_ns,722 ):723 raise ValueError('source changed during fixture/hash preparation')724 else:725 meta = synthetic_fixture(fixture, args.frames, args.seconds)726 identity = dict(kind='generated-synthetic')727 manifest = args.output / 'fixture.json'728 save(manifest, meta)729 report = dict(730 format='platform-benchmark-v1',731 source=identity,732 fixture=meta,733 environment=dict(734 python=sys.version,735 executable=sys.executable,736 os=platform.platform(),737 machine=platform.machine(),738 logical_cpus=os.cpu_count(),739 qt_platform='offscreen',740 cache='uncontrolled; no cache flush',741 git_head=subprocess.check_output(742 ['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True743 ).strip(),744 native_sha256=(745 sha256(ROOT / 'build/libitch_market.dylib')746 if sys.platform == 'darwin'747 else sha256(ROOT / 'build/libitch_market.so')748 ),749 source_files_sha256={750 str(p.relative_to(ROOT)): sha256(p)751 for parent, pattern in (752 (ROOT / 'sim_platform', '*.py'),753 (ROOT / 'src', '*.cpp'),754 (ROOT / 'include', '*.hpp'),755 )756 for p in sorted(parent.rglob(pattern))757 },758 ),759 fixture_and_hash_seconds=time.perf_counter() - preparation_started,760 samples=[],761 summary=[],762 checks={},763 )764 print(765 f"Fixture: {meta['playback_frames']:,} measured frames, "766 f"{(meta['end_time_ns']-meta['origin']['time_ns'])/1e9:.3f} simulated seconds",767 flush=True,768 )769 jobs = []770 rng = random.Random(20261004)771 for repeat in range(args.repeats):772 cases = list(args.cases)773 rng.shuffle(cases)774 jobs.extend((case, repeat + 1, False) for case in cases)775 jobs.extend((case, 1, True) for case in args.profiles if case in args.cases)776 for number, (case, repeat, instrumented) in enumerate(jobs, 1):777 result_path = args.output / f'{number:03d}-{case}.json'778 command = [779 sys.executable,780 str(Path(__file__).resolve()),781 '--worker',782 case,783 '--input',784 str(fixture),785 '--fixture-meta',786 str(manifest),787 '--result',788 str(result_path),789 '--symbol',790 args.symbol,791 '--timeout',792 str(args.timeout),793 ]794 if instrumented:795 command.append('--profile')796 env = dict(os.environ, QT_QPA_PLATFORM='offscreen')797 print(798 f'[{number}/{len(jobs)}] {case}, repeat {repeat}'799 + (' (instrumented)' if instrumented else ''),800 flush=True,801 )802 try:803 process = subprocess.run(804 command, cwd=ROOT, env=env, capture_output=True, text=True, timeout=args.timeout805 )806 if result_path.exists():807 sample = json.loads(result_path.read_text())808 else:809 sample = dict(810 case=case,811 instrumented=instrumented,812 status='failed',813 error=f'child exit {process.returncode}',814 stderr=process.stderr[-4000:],815 )816 except subprocess.TimeoutExpired:817 sample = dict(818 case=case, instrumented=instrumented, status='failed', error='parent trial timeout'819 )820 sample['repeat'] = repeat821 report['samples'].append(sample)822 report['summary'] = aggregate(report['samples'])823 save(args.output / 'report.json', report)824 if sample['status'] == 'completed':825 print(f" {sample['wall_seconds']:.3f}s, {sample['speed_x']:.3f}×", flush=True)826 else:827 print(' FAILED: ' + sample['error'], flush=True)828 valid = {829 s['case']: s830 for s in report['samples']831 if s['status'] == 'completed' and not s['instrumented']832 }833 for left, right in (834 ('native', 'market'),835 ('execution_idle', 'human_idle'),836 ('human_idle', 'rule_idle'),837 ('human_idle', 'desktop_idle'),838 ('rule_trade', 'desktop_trade'),839 ):840 if left in valid and right in valid:841 a, b = valid[left], valid[right]842 if left == 'native':843 equal = a['final']['data'] == {844 k: b['final']['data'][k] for k in ('sequence', 'time_ns', 'bids', 'asks')845 }846 else:847 equal = a['final']['sha256'] == b['final']['sha256']848 report['checks'][left + '=' + right] = equal849 save(args.output / 'report.json', report)850 (args.output / 'report.md').write_text(markdown(report))851 print('Saved ' + str(args.output / 'report.md'), flush=True)852 failed = any(s['status'] != 'completed' for s in report['samples'])853 mismatch = any(not value for value in report['checks'].values())854 nondeterministic = any(not r.get('deterministic', False) for r in report['summary'])855 return 1 if failed or mismatch or nondeterministic else 0856857858def parser():859 result = argparse.ArgumentParser(description=__doc__)860 result.add_argument(861 '--input', type=Path, help='optional real BinaryFILE; otherwise generate synthetic data'862 )863 result.add_argument('--output', type=Path, default=ROOT / 'build/platform-baseline')864 result.add_argument('--symbol', default='AAPL')865 result.add_argument('--seconds', type=float, default=5)866 result.add_argument(867 '--start-time-ns',868 type=int,869 help='real-data origin at/after this source timestamp, with a two-sided book',870 )871 result.add_argument('--frames', type=int, default=1000, help='synthetic playback frames')872 result.add_argument('--repeats', type=int, default=3)873 result.add_argument('--cases', nargs='+', choices=CASES, default=list(CASES))874 result.add_argument(875 '--profiles', nargs='*', choices=CASES, default=['human_idle', 'rule_trade', 'desktop_idle']876 )877 result.add_argument('--timeout', type=float, default=600)878 result.add_argument('--worker', choices=CASES, help=argparse.SUPPRESS)879 result.add_argument('--fixture-meta', type=Path, help=argparse.SUPPRESS)880 result.add_argument('--result', type=Path, help=argparse.SUPPRESS)881 result.add_argument('--profile', action='store_true', help=argparse.SUPPRESS)882 return result883884885if __name__ == '__main__':886 argument_parser = parser()887 options = argument_parser.parse_args()888 import math889890 if (891 not math.isfinite(options.seconds)892 or options.seconds <= 0893 or options.frames < 1894 or options.repeats < 1895 or not math.isfinite(options.timeout)896 or options.timeout <= 0897 ):898 argument_parser.error('seconds, frames, repeats and timeout must be positive and finite')899 sys.exit(worker(options) if options.worker else main(options))