m/market labResearch, with a record.
Source on GitHub ↗

benchmark_experiment_runtime.py

1"""Short Runtime dispatch probe; scheduling timestamps, not whole-day capacity.23Runs real Runtime children under idle and parent Python CPU load. No model,4SQL, network or new benchmark instrumentation in production hot paths.5"""67import argparse8import json9import platform10from pathlib import Path11import statistics12import multiprocessing13from math import ceil14import sys15import time1617ROOT = Path(__file__).resolve().parents[1]18sys.path.insert(0, str(ROOT))19from sim_platform.experiments.scheduling.policy.controller import LiveWorkPolicy20from sim_platform.experiments.scheduling.host import ExperimentRuntimeHandle212223def summary(values):24    values = sorted(values)25    return dict(26        median_ms=statistics.median(values),27        p99_ms=values[ceil(len(values) * 0.99) - 1],28        max_ms=max(values),29    )303132def trial(mode, rounds):33    config = dict(34        start_ns=0,35        end_ns=60_000_000_000,36        symbol='AAPL',37        event_buckets=4,38        schedule=dict(39            mode='interval',40            interval_seconds='0.02',41            busy_event_policy='preserve_interval',42            events=[],43        ),44        timeout_seconds='30',45        retry=dict(46            max_attempts=3,47            format_max_attempts=3,48            base_delay_seconds='1',49            max_delay_seconds='8',50            retry_classes=['timeout'],51        ),52    )53    dispatch, delivery = [], []54    with ExperimentRuntimeHandle(LiveWorkPolicy, config) as runtime:55        runtime.clock.resume()56        work = runtime.submit('next_work').result(2)57        for _ in range(rounds):58            runtime.submit('finish', {k: work[k] for k in ('work_id', 'epoch')}).result(2)59            result = runtime.submit('next_work')60            received = []61            result.add_done_callback(lambda _: received.append(runtime.clock.target_ns()))62            if mode == 'parent_cpu':63                deadline = time.perf_counter() + 0.0664                value = 065                while time.perf_counter() < deadline:66                    value = (value * 3 + 1) % 1_000_00367            work = result.result(2)68            due = int(work['scheduled_due_ns'])69            dispatch.append((int(work['dispatched_ns']) - due) / 1e6)70            # Future completion precedes its callbacks: allow callback adoption.71            deadline = time.monotonic() + 172            while not received:73                if time.monotonic() >= deadline:74                    raise RuntimeError('delivery timestamp callback did not finish')75                time.sleep(0.001)76            delivery.append((received[0] - due) / 1e6)77    return dict(mode=mode, rounds=rounds, dispatch=summary(dispatch), delivery=summary(delivery))787980def direct_agent(client, output, acknowledgement, rounds):81    try:82        work = client.next_work()83        for _ in range(rounds):84            due = client.finish(work)['next_due_ns']85            output.send(dict(due_ns=due))86            work = client.next_work()87            output.send(dict(work=work, received_ns=client.target_ns()))88            acknowledgement.recv()89    finally:90        client.close()91        output.close()92        acknowledgement.close()939495def direct_trial(rounds):96    config = dict(97        start_ns=0,98        end_ns=60_000_000_000,99        symbol='AAPL',100        event_buckets=4,101        schedule=dict(102            mode='interval',103            interval_seconds='0.02',104            busy_event_policy='preserve_interval',105            events=[],106        ),107        timeout_seconds='30',108        retry=dict(109            max_attempts=3,110            format_max_attempts=3,111            base_delay_seconds='1',112            max_delay_seconds='8',113            retry_classes=['timeout'],114        ),115    )116    dispatch, delivery = [], []117    context = multiprocessing.get_context('spawn')118    with ExperimentRuntimeHandle(LiveWorkPolicy, config) as runtime:119        runtime.clock.resume()120        client = runtime.work_client()121        read, write = context.Pipe(duplex=False)122        child_ack, parent_ack = context.Pipe(duplex=False)123        child = context.Process(target=direct_agent, args=(client, write, child_ack, rounds))124        child.start()125        write.close()126        child_ack.close()127        try:128            for _ in range(rounds):129                if not read.poll(2):130                    raise RuntimeError('direct agent preparation timed out')131                due = int(read.recv()['due_ns'])132                deadline = time.perf_counter() + 0.06133                value = 0134                while time.perf_counter() < deadline:135                    value = (value * 3 + 1) % 1_000_003136                if not read.poll(2):137                    raise RuntimeError('direct agent work delivery timed out')138                received = read.recv()139                dispatch.append((int(received['work']['dispatched_ns']) - due) / 1e6)140                delivery.append((received['received_ns'] - due) / 1e6)141                parent_ack.send(True)142        finally:143            child.join(1)144            if child.is_alive():145                child.terminate()146                child.join(1)147            client.close()148            read.close()149            parent_ack.close()150        if child.exitcode:151            raise RuntimeError('direct agent child failed')152    return dict(153        mode='parent_cpu_direct_agent',154        rounds=rounds,155        dispatch=summary(dispatch),156        delivery=summary(delivery),157    )158159160def main():161    parser = argparse.ArgumentParser()162    parser.add_argument('--output', type=Path, required=True)163    args = parser.parse_args()164    results = [trial(mode, 10) for mode in ('idle', 'parent_cpu') for _ in range(3)]165    results.extend(direct_trial(10) for _ in range(3))166    args.output.write_text(167        json.dumps(168            dict(169                python=sys.version,170                machine=platform.machine(),171                os=platform.platform(),172                cases=results,173                limitations='90 short timer samples; nearest-rank p99 with ten samples per trial; '174                'no whole-day, provider or market throughput claim',175            ),176            indent=2,177        )178        + '\n'179    )180181182if __name__ == '__main__':183    main()