benchmark_live_experiments.py
1"""Short native/process load matrix. No real model or account credentials."""23import json4from pathlib import Path5import struct6import sys7import time8from tempfile import TemporaryDirectory9import resource10from functools import partial11from unittest.mock import patch12from concurrent.futures import ThreadPoolExecutor1314sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests/platform'))15sys.path.insert(0, str(Path(__file__).resolve().parents[1]))16from sim_platform import Platform17from sim_platform.decisions.policy.policies import BuyHoldSell18from sim_platform.decisions.policy.models import DecisionConfig19from sim_platform.execution.models import ExecutionConfig2021ORIGIN = 34_200_000_000_000222324def fixture(path, spacing, frames=100):25 def common(kind, stamp):26 return kind.encode() + struct.pack('>HH', 7, 0) + stamp.to_bytes(6, 'big')2728 def frame(value):29 return struct.pack('>H', len(value)) + value3031 records = []32 for n, side in [(1, b'B'), (2, b'S')]:33 records.append(34 frame(35 common('A', ORIGIN)36 + struct.pack('>Q', n)37 + side38 + struct.pack('>I', 100)39 + b'AAPL '40 + struct.pack('>I', 1_000_000 + n)41 )42 )43 for n in range(frames):44 records.append(45 frame(46 common('P', ORIGIN + (n + 1) * spacing)47 + struct.pack('>Q', 0)48 + b'B'49 + struct.pack('>I', 1)50 + b'AAPL '51 + struct.pack('>IQ', 1_000_000, n + 1)52 )53 )54 path.write_bytes(b''.join(records) + b'\0\0')55 return path565758class SlowRule(BuyHoldSell):59 name = 'slow-rule-probe'6061 def decide(self, observation, state, timeout_ms):62 end = time.monotonic() + 0.0863 total = 064 while time.monotonic() < end:65 total += sum(i * i for i in range(500))66 return super().decide(observation, state, timeout_ms)676869def trial(name, spacing, slow=False):70 with TemporaryDirectory() as directory, Platform(Path(directory) / 'db') as app:71 profile = app.create_profile('probe')72 account = app.create_account(profile, 'cash', 100_000_000)73 controller = app.open_research()74 controller.begin_prepare(75 fixture(76 Path(directory) / 'source.bin', spacing, 400 if name == 'dense-slow-json' else 10077 ),78 'AAPL',79 start_mode='first_book',80 )81 while controller.preparation['status'] == 'preparing':82 controller.prepare_tick()83 source = dict(controller.source, execution_mode='live')84 policy = SlowRule() if slow else BuyHoldSell()85 started = time.perf_counter()86 maker, run = app.create_managed_decision_run(87 profile, account, 'AAPL', policy, DecisionConfig(), ExecutionConfig(), source=source88 )89 runner = app._new_live_policy_runner(profile, run, maker, controller.market, policy)90 controller.runner, controller.profile, controller.run_id = runner, profile, run91 setup = time.perf_counter() - started92 if name == 'dense-slow-json':93 original = runner.pipeline.channels.sink9495 def slow_json(value):96 time.sleep(0.005)97 original(value)9899 runner.pipeline.channels.bind_sink(slow_json)100 if name == 'dense-slow-db':101 original = runner.owner.commands._commit102103 def slow_db(*args, **kwargs):104 time.sleep(0.12)105 return original(*args, **kwargs)106107 runner.owner.commands._commit = slow_db108 runner.play()109 deadline = time.monotonic() + 5110 while runner.active and time.monotonic() < deadline:111 time.sleep(0.001)112 if runner.active:113 raise RuntimeError('probe deadline exceeded')114 view = runner.pipeline.snapshot()115 started = time.perf_counter()116 close_error = None117 try:118 runner.close()119 except Exception as exc:120 close_error = type(exc).__name__ + ': ' + str(exc)121 controller.runner = None122 close = time.perf_counter() - started123 return dict(124 case=name,125 spacing_ns=spacing,126 setup_ms=setup * 1000,127 close_ms=close * 1000,128 **view,129 close_error=close_error,130 app_peak_rss=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,131 child_peak_rss=resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss,132 )133134135class ProbeFormatter:136 def __init__(self, binding, slow=False, large=False):137 from sim_platform.agents.tools.reads.formatter import ReadFormatter138139 self.reader = ReadFormatter(binding)140 self.slow, self.large = slow, large141142 def execute(self, captured):143 if self.slow:144 time.sleep(0.08)145 result = self.reader.execute(captured)146 if self.large:147 result['data']['probe_padding'] = 'x' * (256 * 1024)148 return result149150151def tool_trial(name, slow=False, large=False, burst=False):152 from support.agents import definition, settings153 from sim_platform.agents.tools.worker.host import ToolProcessHost154155 with TemporaryDirectory() as directory, Platform(Path(directory) / 'db') as app:156 profile = app.create_profile('probe')157 account = app.create_account(profile, 'cash', 100_000_000)158 app.create_agent(profile, definition())159 config = dict(160 settings(),161 start_ns=str(ORIGIN),162 end_ns=str(ORIGIN + 2_000_000_000),163 lag=dict(threshold_seconds='0.1', window_seconds='0.1', profile_ref='prototype-100ms'),164 )165 formatter = partial(ProbeFormatter, slow=slow, large=large)166 started = time.perf_counter()167 with patch(168 'sim_platform.agents.tools.server.router.ToolProcessHost',169 partial(ToolProcessHost, formatter_factory=formatter),170 ):171 run = app.open_experiment(172 profile,173 account,174 source=fixture(Path(directory) / 'source.bin', 1_000_000, 400),175 method='agent',176 agent_id='demo',177 revision=1,178 settings=config,179 execution_mode='live',180 )181 setup = time.perf_counter() - started182 run.play()183 work = run.runner.next_work().result(2)['result']184185 def query(index):186 request = dict(187 run_id=run.run_id,188 work_id=work['work_id'],189 epoch=work['epoch'],190 request_id='probe-' + str(index),191 tool='market.quote',192 arguments=dict(symbol='AAPL'),193 )194 start = time.perf_counter()195 try:196 response = run.request_tool(request).result(3)['result']197 return dict(198 status=response['status'],199 elapsed_ms=(time.perf_counter() - start) * 1000,200 cutoff=response.get('cutoff'),201 )202 except Exception as exc:203 return dict(204 status=type(exc).__name__, elapsed_ms=(time.perf_counter() - start) * 1000205 )206207 with ThreadPoolExecutor(max_workers=8 if burst else 1) as workers:208 requests = list(workers.map(query, range(24 if burst else 3)))209 deadline = time.monotonic() + 5210 while run.runner.active and time.monotonic() < deadline:211 time.sleep(0.001)212 view = run.runner.pipeline.snapshot()213 started = time.perf_counter()214 run.stop()215 close = time.perf_counter() - started216 return dict(217 case=name,218 setup_ms=setup * 1000,219 close_ms=close * 1000,220 requests=requests,221 **view,222 app_peak_rss=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,223 child_peak_rss=resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss,224 )225226227if __name__ == '__main__':228 values = [229 trial(name, spacing, slow)230 for name, spacing, slow in [231 ('sparse-rule', 3_000_000, False),232 ('dense-rule', 1_000_000, False),233 ('dense-slow-rule', 1_000_000, True),234 ('dense-slow-json', 1_000_000, False),235 ('dense-slow-db', 1_000_000, False),236 ]237 ]238 values.extend(239 [240 tool_trial('dense-slow-query', slow=True),241 tool_trial('dense-large-result', large=True),242 tool_trial('dense-burst', burst=True),243 ]244 )245 result = dict(246 cases=values,247 scope='eight short synthetic native trials, including deliberately overloaded failure cases; RSS bytes are lifetime process high-water, not per-run incremental; instrumentation counted only frame metric updates; no public ranking profile',248 )249 destination = Path(__file__).resolve().parents[1] / 'build/benchmarks/live-experiments.json'250 destination.parent.mkdir(parents=True, exist_ok=True)251 destination.write_text(json.dumps(result, indent=2) + '\n')252 print(json.dumps(result, indent=2))