host.py
1"""One persistent, trusted tools process per run; owner never waits on pipe IO.23Only sender/receiver adapters encode/decode and block on IPC. cancel_pending is4nonblocking; close reaps the child off-owner and retains resources on failure.5No database, market or execution capability is passed to the child.6"""78from __future__ import annotations9import logging1011from concurrent.futures import Future12import json13import multiprocessing14from queue import Queue, Empty15from threading import Thread, RLock, Event16import time17from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Union1819from sim_platform.domain.models import ContractError20from sim_platform.domain.json_values import copy_json21from sim_platform.experiments.execution.results import DeferredResult22from sim_platform.agents.tools.reads.capture import CapturedRead23from sim_platform.agents.tools.trading.financial import format_financial24from sim_platform.agents.tools.reads.formatter import ReadFormatter25from sim_platform.agents.tools.protocol.responses import response_size, error_response26from sim_platform.agents.tools.protocol.contracts import ToolError27from sim_platform.agents.tools.protocol.transport import encode_job2829from .process import run_tools3031_log = logging.getLogger(__name__)3233if TYPE_CHECKING: # Annotation only.34 from sim_platform.agents.tools.trading.financial import FinancialCapture3536 Captured = Union[CapturedRead, FinancialCapture]3738# Child startup poll, IPC poll, and close() join waits (seconds).39READY_POLL_SECONDS = 0.0240IPC_POLL_SECONDS = 0.0541CHILD_JOIN_SECONDS = 0.242FORCED_JOIN_SECONDS = 143THREAD_JOIN_SECONDS = 144# Extra bytes allowed around a response for its envelope, and longest error text.45ENVELOPE_BYTES = 102446ERROR_TEXT_LIMIT = 2564748TOOL_BINDING_FIELDS = (49 'run_id',50 'work_id',51 'epoch',52 'symbol',53 'start_ns',54 'end_ns',55 'source_manifest_ref',56 'execution_config_ref',57 'limits',58 'granted_tools',59)606162class ToolProcessHost:63 """Bounded capture → process → Future adapter; caller retains ordered output slots.6465 AgentTools creates one per run; it starts the tools child and its send/receive66 threads. close() stops and joins them, keeping ownership (raising) if they do67 not end.68 """6970 def __init__(71 self,72 binding: Dict[str, Any],73 *,74 formatter_factory: Callable[[Dict[str, Any]], ReadFormatter] = ReadFormatter,75 startup_timeout: float = 5,76 ) -> None:77 self.binding = copy_json({name: binding[name] for name in TOOL_BINDING_FIELDS})78 self._queue: Queue[Tuple[int, Captured]] = Queue(binding['limits']['pending_requests'])79 self._pending: Dict[int, Tuple[Captured, Future[Any]]] = {}80 self._held: Dict[int, Tuple[Any, Optional[BaseException]]] = {}81 self._lock = RLock()82 self._closed = False83 self._ending = False84 self._error: Optional[str] = None85 self._identity = 086 self._delivery = Event()87 self._delivery.set()88 context = multiprocessing.get_context('spawn')89 self._ready, self._child_stop = context.Event(), context.RawValue('b', 0)90 self._stop = Event()91 child_read, self._write = context.Pipe(duplex=False)92 self._read, child_write = context.Pipe(duplex=False)93 self._process = context.Process(94 target=run_tools,95 name='agent-tools',96 args=(97 child_read,98 child_write,99 self._ready,100 self._child_stop,101 self.binding,102 formatter_factory,103 ),104 )105 self._threads = [106 Thread(target=self._send, name='tools-ipc-send'),107 Thread(target=self._receive, name='tools-ipc-receive'),108 ]109 try:110 started = time.monotonic()111 self._process.start()112 self._pid = self._process.pid113 child_read.close()114 child_write.close()115 while not self._ready.wait(READY_POLL_SECONDS):116 if not self._process.is_alive() or time.monotonic() - started > startup_timeout:117 raise ContractError('tools process did not become ready')118 for thread in self._threads:119 thread.start()120 except Exception:121 child_read.close()122 child_write.close()123 self.close()124 raise125126 def snapshot(self) -> Dict[str, Any]:127 with self._lock:128 return dict(129 pid=getattr(self, '_pid', None),130 alive=self._process.is_alive() if not self._closed else False,131 pending=len(self._pending),132 error=self._error,133 state=(134 'closed'135 if self._closed136 else ('failed' if self._error else ('ending' if self._ending else 'ready'))137 ),138 )139140 def require_available(self) -> None:141 """Owner pre-effect check; no financial action starts after known transport failure."""142 if self._ending or self._error is not None or not self._process.is_alive():143 raise ToolError('TOOL_UNAVAILABLE', 'tools process is unavailable')144 if len(self._pending) >= self._queue.maxsize:145 raise ToolError('QUEUE_FULL', 'tools process capacity exhausted')146147 def submit(self, captured: Captured) -> DeferredResult:148 """Owner enqueues references only: no encoding, pipe send or Future wait."""149 with self._lock:150 try:151 self.require_available()152 except ToolError:153 if not isinstance(captured, CapturedRead):154 from sim_platform.agents.tools.trading.financial import financial_fallback155156 return financial_fallback(captured, self.binding)157 raise158 if len(self._pending) >= self._queue.maxsize:159 raise ToolError('QUEUE_FULL', 'tools process capacity exhausted')160 self._identity += 1161 identity = self._identity162 future: Future[Any] = Future()163 self._pending[identity] = captured, future164 self._queue.put_nowait((identity, captured))165 return DeferredResult(future)166167 def _finish(168 self, identity: int, result: Any = None, error: Optional[BaseException] = None169 ) -> None:170 # Completion and cancellation share the same publication lock. A result171 # cannot disappear from pending just before Stop cancels unfinished reads.172 with self._lock:173 record = self._pending.get(identity)174 if record is None:175 return176 if not self._delivery.is_set() and not self._ending:177 self._held[identity] = result, error178 return179 self._pending.pop(identity)180 self._held.pop(identity, None)181 _, future = record182 if not future.done():183 if error is None:184 future.set_result(result)185 else:186 future.set_exception(error)187188 def _flush_held(self) -> None:189 with self._lock:190 for identity, (result, error) in list(self._held.items()):191 self._finish(identity, result, error)192193 def _send(self) -> None:194 try:195 while not self._stop.is_set():196 try:197 identity, captured = self._queue.get(timeout=IPC_POLL_SECONDS)198 except Empty:199 continue200 with self._lock:201 if identity not in self._pending:202 continue203 try:204 self._write.send_bytes(encode_job(identity, captured))205 except ToolError as error:206 self._finish(207 identity,208 error_response(209 captured.request, captured.receipt_index, dict(captured.position), error210 ),211 )212 except (OSError, EOFError):213 raise214 except Exception as error:215 _log.warning('Handled failure in _send', exc_info=True)216 self._finish(identity, error=error)217 except Exception as error:218 _log.warning('Handled failure in _send', exc_info=True)219 self._failed(str(error))220221 def _receive(self) -> None:222 try:223 maximum = self.binding['limits']['tool_response_bytes'] + ENVELOPE_BYTES224 while not self._stop.is_set():225 if self._ending:226 self._drain_financial()227 if not self._read.poll(IPC_POLL_SECONDS):228 if not self._process.is_alive():229 raise ContractError('tools process exited')230 continue231 message = json.loads(self._read.recv_bytes(maximum))232 if (233 set(message) != {'identity', 'result', 'fault'}234 or type(message['identity']) is not int235 or not 1 <= message['identity'] <= self._identity236 ):237 raise ContractError('invalid tools response envelope')238 while not self._delivery.wait(IPC_POLL_SECONDS):239 if self._stop.is_set():240 return241 fault = message['fault']242 self._finish(243 message['identity'], message['result'], RuntimeError(fault) if fault else None244 )245 except Exception as error:246 _log.warning('Handled failure in _receive', exc_info=True)247 if not self._stop.is_set():248 self._failed(str(error))249250 def _finish_financial(self, identity: int, captured: FinancialCapture) -> None:251 """Fault/terminal-only formatting off-owner; retain truthful committed output."""252 try:253 result = format_financial(captured, self.binding['source_manifest_ref'])254 response_size(result, self.binding['limits']['tool_response_bytes'])255 self._finish(identity, result)256 except Exception as error:257 _log.warning('Handled failure in _finish_financial', exc_info=True)258 self._finish(identity, error=error)259260 def _drain_financial(self) -> None:261 with self._lock:262 records = list(self._pending.items())263 for identity, (captured, _) in records:264 if not isinstance(captured, CapturedRead):265 self._finish_financial(identity, captured)266267 def _failed(self, reason: str) -> None:268 with self._lock:269 if self._error is None:270 self._error = reason[:ERROR_TEXT_LIMIT]271 records = list(self._pending.items())272 for identity, (captured, _) in records:273 if isinstance(captured, CapturedRead):274 self._finish(275 identity,276 error_response(277 captured.request,278 captured.receipt_index,279 dict(captured.position),280 ToolError('TOOL_UNAVAILABLE', 'tools process exited'),281 ),282 )283 else:284 # Output-only recovery of a retained outcome, never a resubmission.285 self._finish_financial(identity, captured)286 self._stop.set()287 self._child_stop.value = 1288289 def pause(self) -> Dict[str, Any]:290 with self._lock:291 self._delivery.clear()292 return dict(participant='tools', phase='gate_closed', pid=self._process.pid)293294 def play(self) -> Dict[str, Any]:295 with self._lock:296 self._delivery.set()297 self._flush_held()298 return dict(participant='tools', phase='gate_open', pid=self._process.pid)299300 def cancel_pending(self) -> None:301 """Owner terminal hook: cancel pure reads only, never join/kill/wait here."""302 with self._lock:303 self._ending = True304 for identity, (captured, future) in list(self._pending.items()):305 if isinstance(captured, CapturedRead):306 future.cancel()307 self._pending.pop(identity)308 self._held.pop(identity, None)309 self._delivery.set() # Already committed financial output must drain.310 self._flush_held()311312 def close(self) -> None:313 """Called after pipeline output drains; terminate stuck work without orphaning it."""314 if self._closed:315 return316 self.cancel_pending()317 self._stop.set()318 self._child_stop.value = 1319 self._delivery.set()320 if self._process.pid is not None:321 self._process.join(CHILD_JOIN_SECONDS)322 if self._process.is_alive():323 self._process.terminate()324 self._process.join(FORCED_JOIN_SECONDS)325 if self._process.is_alive():326 self._process.kill()327 self._process.join(FORCED_JOIN_SECONDS)328 if self._process.is_alive():329 raise ContractError('tools process did not exit')330 for thread in self._threads:331 if thread.ident is not None:332 thread.join(THREAD_JOIN_SECONDS)333 if any(thread.is_alive() for thread in self._threads):334 raise ContractError('tools IPC adapters did not close')335 self._write.close()336 self._read.close()337 with self._lock:338 self._process.close()339 self._closed = True340 self._pending.clear()341 self._held.clear()342 while not self._queue.empty():343 self._queue.get_nowait()