benchmark_experiment_runtime.py
1"""Short Runtime dispatch probe; scheduling timestamps, not whole-day capacity.23Runs real Runtime children under idle and parent Python CPU load. No model,4SQL, network or new benchmark instrumentation in production hot paths.5"""67import argparse8import json9import platform10from pathlib import Path11import statistics12import multiprocessing13from math import ceil14import sys15import time1617ROOT = Path(__file__).resolve().parents[1]18sys.path.insert(0, str(ROOT))19from sim_platform.experiments.scheduling.policy.controller import LiveWorkPolicy20from sim_platform.experiments.scheduling.host import ExperimentRuntimeHandle212223def summary(values):24 values = sorted(values)25 return dict(26 median_ms=statistics.median(values),27 p99_ms=values[ceil(len(values) * 0.99) - 1],28 max_ms=max(values),29 )303132def trial(mode, rounds):33 config = dict(34 start_ns=0,35 end_ns=60_000_000_000,36 symbol='AAPL',37 event_buckets=4,38 schedule=dict(39 mode='interval',40 interval_seconds='0.02',41 busy_event_policy='preserve_interval',42 events=[],43 ),44 timeout_seconds='30',45 retry=dict(46 max_attempts=3,47 format_max_attempts=3,48 base_delay_seconds='1',49 max_delay_seconds='8',50 retry_classes=['timeout'],51 ),52 )53 dispatch, delivery = [], []54 with ExperimentRuntimeHandle(LiveWorkPolicy, config) as runtime:55 runtime.clock.resume()56 work = runtime.submit('next_work').result(2)57 for _ in range(rounds):58 runtime.submit('finish', {k: work[k] for k in ('work_id', 'epoch')}).result(2)59 result = runtime.submit('next_work')60 received = []61 result.add_done_callback(lambda _: received.append(runtime.clock.target_ns()))62 if mode == 'parent_cpu':63 deadline = time.perf_counter() + 0.0664 value = 065 while time.perf_counter() < deadline:66 value = (value * 3 + 1) % 1_000_00367 work = result.result(2)68 due = int(work['scheduled_due_ns'])69 dispatch.append((int(work['dispatched_ns']) - due) / 1e6)70 # Future completion precedes its callbacks: allow callback adoption.71 deadline = time.monotonic() + 172 while not received:73 if time.monotonic() >= deadline:74 raise RuntimeError('delivery timestamp callback did not finish')75 time.sleep(0.001)76 delivery.append((received[0] - due) / 1e6)77 return dict(mode=mode, rounds=rounds, dispatch=summary(dispatch), delivery=summary(delivery))787980def direct_agent(client, output, acknowledgement, rounds):81 try:82 work = client.next_work()83 for _ in range(rounds):84 due = client.finish(work)['next_due_ns']85 output.send(dict(due_ns=due))86 work = client.next_work()87 output.send(dict(work=work, received_ns=client.target_ns()))88 acknowledgement.recv()89 finally:90 client.close()91 output.close()92 acknowledgement.close()939495def direct_trial(rounds):96 config = dict(97 start_ns=0,98 end_ns=60_000_000_000,99 symbol='AAPL',100 event_buckets=4,101 schedule=dict(102 mode='interval',103 interval_seconds='0.02',104 busy_event_policy='preserve_interval',105 events=[],106 ),107 timeout_seconds='30',108 retry=dict(109 max_attempts=3,110 format_max_attempts=3,111 base_delay_seconds='1',112 max_delay_seconds='8',113 retry_classes=['timeout'],114 ),115 )116 dispatch, delivery = [], []117 context = multiprocessing.get_context('spawn')118 with ExperimentRuntimeHandle(LiveWorkPolicy, config) as runtime:119 runtime.clock.resume()120 client = runtime.work_client()121 read, write = context.Pipe(duplex=False)122 child_ack, parent_ack = context.Pipe(duplex=False)123 child = context.Process(target=direct_agent, args=(client, write, child_ack, rounds))124 child.start()125 write.close()126 child_ack.close()127 try:128 for _ in range(rounds):129 if not read.poll(2):130 raise RuntimeError('direct agent preparation timed out')131 due = int(read.recv()['due_ns'])132 deadline = time.perf_counter() + 0.06133 value = 0134 while time.perf_counter() < deadline:135 value = (value * 3 + 1) % 1_000_003136 if not read.poll(2):137 raise RuntimeError('direct agent work delivery timed out')138 received = read.recv()139 dispatch.append((int(received['work']['dispatched_ns']) - due) / 1e6)140 delivery.append((received['received_ns'] - due) / 1e6)141 parent_ack.send(True)142 finally:143 child.join(1)144 if child.is_alive():145 child.terminate()146 child.join(1)147 client.close()148 read.close()149 parent_ack.close()150 if child.exitcode:151 raise RuntimeError('direct agent child failed')152 return dict(153 mode='parent_cpu_direct_agent',154 rounds=rounds,155 dispatch=summary(dispatch),156 delivery=summary(delivery),157 )158159160def main():161 parser = argparse.ArgumentParser()162 parser.add_argument('--output', type=Path, required=True)163 args = parser.parse_args()164 results = [trial(mode, 10) for mode in ('idle', 'parent_cpu') for _ in range(3)]165 results.extend(direct_trial(10) for _ in range(3))166 args.output.write_text(167 json.dumps(168 dict(169 python=sys.version,170 machine=platform.machine(),171 os=platform.platform(),172 cases=results,173 limitations='90 short timer samples; nearest-rank p99 with ten samples per trial; '174 'no whole-day, provider or market throughput claim',175 ),176 indent=2,177 )178 + '\n'179 )180181182if __name__ == '__main__':183 main()