m/market labResearch, with a record.
Source on GitHub ↗

benchmark_desktop_controls.py

1"""Measure queued Pause/Stop acknowledgements in the current HTML Desktop."""23import argparse4import json5import os6from pathlib import Path7import sys8from tempfile import TemporaryDirectory9import time1011ROOT = Path(__file__).resolve().parents[1]12sys.path.insert(0, str(ROOT))13from tools.benchmark_platform import percentile, save, sha25614from tools.desktop.driver import DesktopDriver151617def main(args):18    os.environ.setdefault('QT_QPA_PLATFORM', 'offscreen')19    meta = json.loads((args.fixture_dir / 'fixture.json').read_text())20    source = args.fixture_dir / 'prefix.bin'21    if sha256(source) != meta['sha256']:22        raise ValueError('fixture digest mismatch')23    samples = []24    for policy in ('human', 'rule'):25        for repeat in range(args.repeats):26            with TemporaryDirectory(prefix='desktop-controls-') as temporary:27                driver = DesktopDriver(28                    Path(temporary) / 'accounts', (source, None), timeout=args.timeout29                )30                try:31                    driver.prepare(source, policy=policy, start_ns=meta['origin']['time_ns'])32                    driver.operation('play', speed=None)33                    target = meta['origin']['sequence'] + max(34                        1, int(meta['playback_frames'] * args.at_fraction)35                    )36                    driver.until(lambda: driver.view['market']['sequence'] >= target)37                    if driver.view['execution']['status'] != 'running':38                        raise ValueError('fixture ended before Pause probe; use a longer fixture')39                    started = time.perf_counter()40                    driver.operation('pause')41                    driver.until(lambda: driver.view['market']['status'] == 'paused')42                    paused = time.perf_counter() - started43                    sequence = driver.view['market']['sequence']44                    end = time.monotonic() + 0.145                    driver.until(lambda: time.monotonic() >= end)46                    if driver.view['market']['sequence'] != sequence:47                        raise ValueError('source advanced after Pause acknowledgement')48                    driver.operation('play', speed=None)49                    driver.until(lambda: driver.view['market']['sequence'] > sequence)50                    started = time.perf_counter()51                    driver.operation('stop')52                    driver.until(lambda: driver.view['execution']['status'] == 'stopped')53                    stopped = time.perf_counter() - started54                    if (55                        driver.view['execution']['reserved_cash']56                        or driver.view['execution']['reserved_shares']57                    ):58                        raise ValueError('Stop did not release reservations')59                    samples.append(60                        dict(61                            policy=policy,62                            repeat=repeat + 1,63                            pause_seconds=paused,64                            stop_seconds=stopped,65                            paused_sequence=sequence,66                            stopped_sequence=driver.view['market']['sequence'],67                            pause_does_not_advance=True,68                            stop_releases_reservations=True,69                        )70                    )71                finally:72                    started = time.perf_counter()73                    driver.close()74                    if (75                        samples76                        and samples[-1]['policy'] == policy77                        and samples[-1]['repeat'] == repeat + 178                    ):79                        samples[-1]['close_seconds'] = time.perf_counter() - started80    save(81        args.output,82        dict(83            format='desktop-control-baseline-v2',84            fixture_sha256=meta['sha256'],85            qt_platform=os.environ['QT_QPA_PLATFORM'],86            samples=samples,87            note='Acknowledgement samples, not a renderer paint latency or established SLO.',88            summary={89                policy: {90                    key: dict(91                        p50=percentile([s[key] for s in samples if s['policy'] == policy], 0.5),92                        p95=percentile([s[key] for s in samples if s['policy'] == policy], 0.95),93                    )94                    for key in ('pause_seconds', 'stop_seconds', 'close_seconds')95                }96                for policy in ('human', 'rule')97            },98        ),99    )100101102if __name__ == '__main__':103    parser = argparse.ArgumentParser(description=__doc__)104    parser.add_argument('--fixture-dir', type=Path, required=True)105    parser.add_argument('--output', type=Path, required=True)106    parser.add_argument('--repeats', type=int, default=3)107    parser.add_argument('--timeout', type=float, default=120)108    parser.add_argument('--at-fraction', type=float, default=0.1)109    args = parser.parse_args()110    if args.repeats < 1 or args.timeout <= 0 or not 0 < args.at_fraction < 1:111        parser.error('invalid measurement options')112    if args.output.exists():113        parser.error('output already exists')114    main(args)