m/market labResearch, with a record.
Source on GitHub ↗

benchmark_live_experiments.py

1"""Short native/process load matrix. No real model or account credentials."""23import json4from pathlib import Path5import struct6import sys7import time8from tempfile import TemporaryDirectory9import resource10from functools import partial11from unittest.mock import patch12from concurrent.futures import ThreadPoolExecutor1314sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests/platform'))15sys.path.insert(0, str(Path(__file__).resolve().parents[1]))16from sim_platform import Platform17from sim_platform.decisions.policy.policies import BuyHoldSell18from sim_platform.decisions.policy.models import DecisionConfig19from sim_platform.execution.models import ExecutionConfig2021ORIGIN = 34_200_000_000_000222324def fixture(path, spacing, frames=100):25    def common(kind, stamp):26        return kind.encode() + struct.pack('>HH', 7, 0) + stamp.to_bytes(6, 'big')2728    def frame(value):29        return struct.pack('>H', len(value)) + value3031    records = []32    for n, side in [(1, b'B'), (2, b'S')]:33        records.append(34            frame(35                common('A', ORIGIN)36                + struct.pack('>Q', n)37                + side38                + struct.pack('>I', 100)39                + b'AAPL    '40                + struct.pack('>I', 1_000_000 + n)41            )42        )43    for n in range(frames):44        records.append(45            frame(46                common('P', ORIGIN + (n + 1) * spacing)47                + struct.pack('>Q', 0)48                + b'B'49                + struct.pack('>I', 1)50                + b'AAPL    '51                + struct.pack('>IQ', 1_000_000, n + 1)52            )53        )54    path.write_bytes(b''.join(records) + b'\0\0')55    return path565758class SlowRule(BuyHoldSell):59    name = 'slow-rule-probe'6061    def decide(self, observation, state, timeout_ms):62        end = time.monotonic() + 0.0863        total = 064        while time.monotonic() < end:65            total += sum(i * i for i in range(500))66        return super().decide(observation, state, timeout_ms)676869def trial(name, spacing, slow=False):70    with TemporaryDirectory() as directory, Platform(Path(directory) / 'db') as app:71        profile = app.create_profile('probe')72        account = app.create_account(profile, 'cash', 100_000_000)73        controller = app.open_research()74        controller.begin_prepare(75            fixture(76                Path(directory) / 'source.bin', spacing, 400 if name == 'dense-slow-json' else 10077            ),78            'AAPL',79            start_mode='first_book',80        )81        while controller.preparation['status'] == 'preparing':82            controller.prepare_tick()83        source = dict(controller.source, execution_mode='live')84        policy = SlowRule() if slow else BuyHoldSell()85        started = time.perf_counter()86        maker, run = app.create_managed_decision_run(87            profile, account, 'AAPL', policy, DecisionConfig(), ExecutionConfig(), source=source88        )89        runner = app._new_live_policy_runner(profile, run, maker, controller.market, policy)90        controller.runner, controller.profile, controller.run_id = runner, profile, run91        setup = time.perf_counter() - started92        if name == 'dense-slow-json':93            original = runner.pipeline.channels.sink9495            def slow_json(value):96                time.sleep(0.005)97                original(value)9899            runner.pipeline.channels.bind_sink(slow_json)100        if name == 'dense-slow-db':101            original = runner.owner.commands._commit102103            def slow_db(*args, **kwargs):104                time.sleep(0.12)105                return original(*args, **kwargs)106107            runner.owner.commands._commit = slow_db108        runner.play()109        deadline = time.monotonic() + 5110        while runner.active and time.monotonic() < deadline:111            time.sleep(0.001)112        if runner.active:113            raise RuntimeError('probe deadline exceeded')114        view = runner.pipeline.snapshot()115        started = time.perf_counter()116        close_error = None117        try:118            runner.close()119        except Exception as exc:120            close_error = type(exc).__name__ + ': ' + str(exc)121            controller.runner = None122        close = time.perf_counter() - started123        return dict(124            case=name,125            spacing_ns=spacing,126            setup_ms=setup * 1000,127            close_ms=close * 1000,128            **view,129            close_error=close_error,130            app_peak_rss=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,131            child_peak_rss=resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss,132        )133134135class ProbeFormatter:136    def __init__(self, binding, slow=False, large=False):137        from sim_platform.agents.tools.reads.formatter import ReadFormatter138139        self.reader = ReadFormatter(binding)140        self.slow, self.large = slow, large141142    def execute(self, captured):143        if self.slow:144            time.sleep(0.08)145        result = self.reader.execute(captured)146        if self.large:147            result['data']['probe_padding'] = 'x' * (256 * 1024)148        return result149150151def tool_trial(name, slow=False, large=False, burst=False):152    from support.agents import definition, settings153    from sim_platform.agents.tools.worker.host import ToolProcessHost154155    with TemporaryDirectory() as directory, Platform(Path(directory) / 'db') as app:156        profile = app.create_profile('probe')157        account = app.create_account(profile, 'cash', 100_000_000)158        app.create_agent(profile, definition())159        config = dict(160            settings(),161            start_ns=str(ORIGIN),162            end_ns=str(ORIGIN + 2_000_000_000),163            lag=dict(threshold_seconds='0.1', window_seconds='0.1', profile_ref='prototype-100ms'),164        )165        formatter = partial(ProbeFormatter, slow=slow, large=large)166        started = time.perf_counter()167        with patch(168            'sim_platform.agents.tools.server.router.ToolProcessHost',169            partial(ToolProcessHost, formatter_factory=formatter),170        ):171            run = app.open_experiment(172                profile,173                account,174                source=fixture(Path(directory) / 'source.bin', 1_000_000, 400),175                method='agent',176                agent_id='demo',177                revision=1,178                settings=config,179                execution_mode='live',180            )181        setup = time.perf_counter() - started182        run.play()183        work = run.runner.next_work().result(2)['result']184185        def query(index):186            request = dict(187                run_id=run.run_id,188                work_id=work['work_id'],189                epoch=work['epoch'],190                request_id='probe-' + str(index),191                tool='market.quote',192                arguments=dict(symbol='AAPL'),193            )194            start = time.perf_counter()195            try:196                response = run.request_tool(request).result(3)['result']197                return dict(198                    status=response['status'],199                    elapsed_ms=(time.perf_counter() - start) * 1000,200                    cutoff=response.get('cutoff'),201                )202            except Exception as exc:203                return dict(204                    status=type(exc).__name__, elapsed_ms=(time.perf_counter() - start) * 1000205                )206207        with ThreadPoolExecutor(max_workers=8 if burst else 1) as workers:208            requests = list(workers.map(query, range(24 if burst else 3)))209        deadline = time.monotonic() + 5210        while run.runner.active and time.monotonic() < deadline:211            time.sleep(0.001)212        view = run.runner.pipeline.snapshot()213        started = time.perf_counter()214        run.stop()215        close = time.perf_counter() - started216        return dict(217            case=name,218            setup_ms=setup * 1000,219            close_ms=close * 1000,220            requests=requests,221            **view,222            app_peak_rss=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,223            child_peak_rss=resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss,224        )225226227if __name__ == '__main__':228    values = [229        trial(name, spacing, slow)230        for name, spacing, slow in [231            ('sparse-rule', 3_000_000, False),232            ('dense-rule', 1_000_000, False),233            ('dense-slow-rule', 1_000_000, True),234            ('dense-slow-json', 1_000_000, False),235            ('dense-slow-db', 1_000_000, False),236        ]237    ]238    values.extend(239        [240            tool_trial('dense-slow-query', slow=True),241            tool_trial('dense-large-result', large=True),242            tool_trial('dense-burst', burst=True),243        ]244    )245    result = dict(246        cases=values,247        scope='eight short synthetic native trials, including deliberately overloaded failure cases; RSS bytes are lifetime process high-water, not per-run incremental; instrumentation counted only frame metric updates; no public ranking profile',248    )249    destination = Path(__file__).resolve().parents[1] / 'build/benchmarks/live-experiments.json'250    destination.parent.mkdir(parents=True, exist_ok=True)251    destination.write_text(json.dumps(result, indent=2) + '\n')252    print(json.dumps(result, indent=2))