m/market labResearch, with a record.
Source on GitHub ↗

benchmark_prepared_runtime.py

1"""Prepared-runtime baseline and Agent integration performance gate; actual Platform/Runtime/financial UOW.23Uses a fixed predeclared profile, bounded synthetic source, mock/local IO only.4Deployment evidence combines these results with real-source and regression gates.5"""67from pathlib import Path8import argparse9import asyncio10import multiprocessing11import json12import statistics13import sys14import tempfile15import time16from unittest.mock import patch1718ROOT = Path(__file__).resolve().parents[1]19sys.path.insert(0, str(ROOT))20sys.path.insert(0, str(ROOT / 'tests'))21sys.path.insert(0, str(ROOT / 'tests' / 'platform'))22from itch_fixtures import directory, add23from sim_platform import Platform242526def distribution(values):27    if not values:28        return dict(count=0, median_ms=None, p99_ms=None, max_ms=None)29    values = sorted(values)30    return dict(31        count=len(values),32        median_ms=statistics.median(values) / 1e6,33        p99_ms=values[min(len(values) - 1, int(0.99 * len(values)))] / 1e6,34        max_ms=max(values) / 1e6,35    )363738def fixture(path, profile):39    period = 1_000_000_000 // profile['source_events_per_second']40    count = profile['case_seconds'] * profile['source_events_per_second']41    with path.open('wb') as output:42        output.write(directory('AAPL', timestamp=1))43        output.write(add(1, 'B', 100, 99, 'AAPL', timestamp=2))44        output.write(add(2, 'S', 100, 101, 'AAPL', timestamp=3))45        for index in range(count):46            output.write(add(3 + index, 'S', 1, 101, 'AAPL', timestamp=3 + (index + 1) * period))47        output.write(b'\0\0')48    return path495051def run_case(workspace, source, profile, case, trial):52    from sim_platform.experiments.execution.metrics import RuntimeMetrics5354    with Platform(workspace) as app:55        user = app.create_profile('runtime gate')56        account = app.create_account(user, 'cash', 100_000_000)57        experiment = app.open_experiment(user, account, source=source, execution_mode='live')58        pipeline = experiment.runner.pipeline59        frames, commands, queue_wait = [], [], []6061        class Measured(RuntimeMetrics):62            def frame(self, target, event):63                frames.append(max(0, target - event))64                super().frame(target, event)6566            def span(self, target, event, count):67                frames.extend([max(0, target - event)] * count)68                super().span(target, event, count)6970        pipeline.metrics = Measured(100_000_000)71        original = pipeline.operation7273        def timed(request, position):74            queue_wait.append(max(0, pipeline.clock.target_ns() - int(position['time_ns'])))75            return original(request, position)7677        pipeline.operation = timed78        queries = []79        experiment.play()80        start = time.monotonic()81        next_order = start + 0.182        next_read = start + 0.0583        deadline = start + profile['case_seconds']84        number = 085        while time.monotonic() < deadline and experiment.runner.active:86            now = time.monotonic()87            if case != 'idle' and now >= next_order:88                number += 189                next_order += 1 / profile['orders_per_second']90                begun = time.perf_counter_ns()91                experiment.runner.act(92                    [93                        dict(94                            kind='submit',95                            intent=dict(96                                client_id='gate-' + str(number),97                                symbol='AAPL',98                                side='buy',99                                quantity=1,100                                limit_price=101 if case == 'fills_and_reads' else 1,101                            ),102                        )103                    ]104                )105                commands.append(time.perf_counter_ns() - begun)106            if case != 'idle' and now >= next_read:107                next_read += 1 / profile['market_queries_per_second']108                begun = time.perf_counter_ns()109                cutoff = min(pipeline.clock.target_ns(), pipeline.reader.manifest['last_ns'])110                pipeline.reader.snapshot_at(cutoff)111                queries.append(time.perf_counter_ns() - begun)112            time.sleep(0.001)113        until = time.monotonic() + 2114        while experiment.runner.active and time.monotonic() < until:115            time.sleep(0.002)116        state = pipeline.snapshot()117        begun = time.monotonic()118        experiment.close()119        shutdown = time.monotonic() - begun120        thresholds = profile['thresholds']121        delay = distribution(frames)122        command = distribution(commands)123        query = distribution(queries)124        complete = pipeline.driver.sequence == pipeline.reader.manifest['frames']125        passed = (126            complete127            and state['state'] == 'completed'128            and delay['p99_ms'] <= thresholds['market_processing_delay_p99_ms']129            and (not commands or command['p99_ms'] <= thresholds['command_roundtrip_p99_ms'])130            and (not queries or query['p99_ms'] <= thresholds['market_query_roundtrip_p99_ms'])131            and shutdown <= thresholds['shutdown_seconds_max']132        )133        return dict(134            case=case,135            trial=trial,136            passed=passed,137            completed=complete,138            state=state['state'],139            financial_commands=command,140            market_lookup=query,141            owner_queue_wait=distribution(queue_wait),142            market_processing_delay=delay,143            shutdown_seconds=shutdown,144            query_scope='trusted indexed lookup, not complete Tools IPC; Agent tool roundtrip is a separate gate',145        )146147148class ParsingProvider:149    async def request_received(self, profile, payload, secret, received):150        await asyncio.sleep(0.02)151        received()152        # CPU parsing/load is isolated in Agent, not sleeping on the App GIL.153        deadline = time.monotonic() + 0.2154        value = 1155        while time.monotonic() < deadline:156            value = (value * 1664525 + 1013904223) % 2**32157        return dict(output=[], sample='x' * 50000)158159160class ModelWorkload:161    def __init__(self, finished):162        self.finished = finished163164    def __call__(self, client, configuration):165        work = client.next_work()166        result = client.request_model(work, {'input': 'local performance fixture'})167        if result['output'] != []:168            raise RuntimeError('unexpected fixture model result')169        self.finished.value = 1170        client.wait_stopped()171172173def agent_case(workspace, source, profile, trial):174    from functools import partial175    from support.agents import definition, settings176    from support.tools import request177    from sim_platform.agents.hosting.host import AgentProcessHost178    from sim_platform.models.contracts import ModelProfile179    import threading180181    finished = multiprocessing.get_context('spawn').RawValue('b', 0)182    with Platform(workspace) as app:183        user = app.create_profile('Agent gate')184        account = app.create_account(user, 'cash', 100_000_000)185        app.create_agent(user, definition())186        config = dict(187            settings(),188            start_ns='3',189            end_ns=str(3 + profile['case_seconds'] * 1_000_000_000),190            timeout_seconds='2',191            granted_tools=['market.quote', 'orders.submit_limit'],192        )193        with patch(194            'sim_platform.agents.hosting.host.AgentProcessHost',195            partial(AgentProcessHost, program=ModelWorkload(finished)),196        ):197            experiment = app.open_experiment(198                user,199                account,200                source=source,201                method='agent',202                agent_id='demo',203                revision=1,204                settings=config,205                execution_mode='live',206            )207        app.set_model_credential('gate', 'local-fixture-not-an-api-key')208        app.configure_experiment_model(209            experiment,210            ModelProfile('openai-responses', 'mock', 'gate'),211            provider=ParsingProvider(),212        )213        pipeline = experiment.runner.pipeline214        from sim_platform.experiments.execution.metrics import RuntimeMetrics215216        delays = []217218        class Measured(RuntimeMetrics):219            def frame(self, target, event):220                delays.append(max(0, target - event))221                super().frame(target, event)222223            def span(self, target, event, count):224                delays.extend([max(0, target - event)] * count)225                super().span(target, event, count)226227        pipeline.metrics = Measured(100_000_000)228        queries = []229        commands = []230        number = 0231        experiment.play()232        start = time.monotonic()233        next_read = start + 0.05234        next_order = start + 0.1235        while time.monotonic() - start < profile['case_seconds'] and experiment.runner.active:236            work = experiment.runner._tools.controls.snapshot()237            if work['state'] != 'running':238                time.sleep(0.001)239                continue240            now = time.monotonic()241            if now >= next_read:242                number += 1243                next_read += 1 / profile['market_queries_per_second']244                value = request(245                    'market.quote',246                    {'symbol': 'AAPL'},247                    number='read-' + str(number),248                    run=experiment.run_id,249                )250                value.update(work_id=work['work_id'], epoch=work['epoch'])251                begun = time.perf_counter_ns()252                response = experiment.request_tool(value).result(2)['result']253                if response['status'] != 'ok':254                    raise RuntimeError('query failed')255                queries.append(time.perf_counter_ns() - begun)256            if now >= next_order:257                number += 1258                next_order += 1 / profile['orders_per_second']259                value = request(260                    'orders.submit_limit',261                    dict(symbol='AAPL', side='buy', quantity_shares=1, limit_price_usd='0.0001'),262                    number='order-' + str(number),263                    run=experiment.run_id,264                )265                value.update(work_id=work['work_id'], epoch=work['epoch'])266                begun = time.perf_counter_ns()267                response = experiment.request_tool(value).result(2)['result']268                if response['status'] != 'ok':269                    raise RuntimeError('order failed')270                commands.append(time.perf_counter_ns() - begun)271            time.sleep(0.001)272        until = time.monotonic() + 2273        while experiment.runner.active and time.monotonic() < until:274            time.sleep(0.002)275        state = pipeline.snapshot()276        begun = time.monotonic()277        experiment.close()278        shutdown = time.monotonic() - begun279        q = distribution(queries)280        c = distribution(commands)281        d = distribution(delays)282        threshold = profile['thresholds']283        no_model_thread = not any(t.name == 'model-async' for t in threading.enumerate())284        passed = (285            bool(finished.value)286            and state['state'] == 'completed'287            and no_model_thread288            and len(queries) > 0289            and q['p99_ms'] <= threshold['market_query_roundtrip_p99_ms']290            and len(commands) > 0291            and c['p99_ms'] <= threshold['command_roundtrip_p99_ms']292            and d['p99_ms'] <= threshold['market_processing_delay_p99_ms']293            and shutdown <= threshold['shutdown_seconds_max']294        )295        return dict(296            case='Agent model IO/CPU parsing + complete Tools IPC',297            trial=trial,298            passed=passed,299            model_completed=bool(finished.value),300            no_app_model_thread=no_model_thread,301            market_query_roundtrip=q,302            financial_tool_roundtrip=c,303            market_processing_delay=d,304            state=state['state'],305            shutdown_seconds=shutdown,306        )307308309def main():310    parser = argparse.ArgumentParser()311    parser.add_argument('--output', type=Path, required=True)312    parser.add_argument(313        '--include-agent', action='store_true', help='Also measure Agent model IO and tool load.'314    )315    args = parser.parse_args()316    profile = json.loads((ROOT / 'tools/fixtures/prepared-runtime.json').read_text())317    report = dict(318        include_agent=args.include_agent,319        profile=profile,320        deployment='App dedicated execution thread',321        cases=[],322        complete=False,323        passed=False,324    )325    with tempfile.TemporaryDirectory(prefix='prepared-runtime-gate-') as directory:326        root = Path(directory)327        source = fixture(root / 'source.bin', profile)328        for case in profile['cases']:329            for trial in range(profile['repetitions']):330                result = run_case(root / 'app', source, profile, case, trial + 1)331                report['cases'].append(result)332                args.output.write_text(json.dumps(report, indent=2) + '\n')333                print(case, trial + 1, result['passed'], flush=True)334        if args.include_agent:335            report['agent_cases'] = []336            for trial in range(profile['repetitions']):337                result = agent_case(root / ('agent-' + str(trial + 1)), source, profile, trial + 1)338                report['agent_cases'].append(result)339                args.output.write_text(json.dumps(report, indent=2) + '\n')340                print('Agent model/Tools', trial + 1, result['passed'], flush=True)341    report['complete'] = True342    report['passed'] = all(case['passed'] for case in report['cases']) and all(343        case['passed'] for case in report.get('agent_cases', [])344    )345    args.output.write_text(json.dumps(report, indent=2) + '\n')346347348if __name__ == '__main__':349    main()