host.py
1"""Owned Runtime child with bounded nonblocking admission and detached replies.23Only sender/receiver threads wait on pipes; submit never blocks a market owner.4close is an off-owner operation, retains ownership if cleanup is unconfirmed.5The handle is infrastructure, not a second user-facing experiment API.6"""78from __future__ import annotations9import logging1011from concurrent.futures import Future12import multiprocessing13from queue import Queue, Empty14from threading import Thread, RLock, Event15import time16from math import isfinite17from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Set1819from sim_platform.domain.json_values import copy_json20from sim_platform.domain.models import ContractError, integer21from sim_platform.resources import cleanup_after_error22from sim_platform.experiments.scheduling.clock import SharedExperimentClock, StopSignal23from sim_platform.experiments.scheduling.protocol import (24 MAX_BYTES,25 packet,26 unpack,27 validate_payload,28 reply,29)30from sim_platform.experiments.scheduling.worker import run_runtime31from sim_platform.experiments.scheduling.client import RuntimeClient3233_log = logging.getLogger(__name__)3435if TYPE_CHECKING: # Annotation only.36 from multiprocessing.connection import Connection37 from multiprocessing.process import BaseProcess3839 from sim_platform.experiments.scheduling.protocol import Message40 from sim_platform.experiments.scheduling.worker import NoticeCallback, PolicyFactory4142# Admission bounds: default and largest request capacity, and queued controls.43DEFAULT_CAPACITY = 6444MAX_CAPACITY = 102445MAX_PENDING_CONTROLS = 446# Child startup: default and longest wait for 'ready', and the poll interval.47DEFAULT_STARTUP_TIMEOUT_SECONDS = 548MAX_STARTUP_TIMEOUT_SECONDS = 6049READY_POLL_SECONDS = 0.0150# Sender threads re-check the stop flag at this interval.51SEND_POLL_SECONDS = 0.0552# close(): wait per termination step for the child, and for each IO thread.53PROCESS_JOIN_SECONDS = 0.554THREAD_JOIN_SECONDS = 15556# Receives a Runtime failure.57FailureCallback = Callable[[Exception], None]585960class RuntimeError(ContractError):61 """A Runtime refusal with a stable ``code`` (for example 'QUEUE_FULL').6263 Note: this name shadows the built-in RuntimeError inside this module.64 """6566 def __init__(self, code: str, message: str) -> None:67 self.code = code68 super().__init__(message)697071class ExperimentRuntimeHandle:72 """The App side of one Runtime child process and its pipes.7374 Created during experiment assembly; it starts the child, its sender/receiver75 threads, and waits for 'ready'. submit() never blocks; replies arrive on the76 receiver thread. close() stops and joins everything and raises (keeping77 ownership) if the child or a thread does not end.7879 Args:80 factory: Builds the work policy inside the child.81 configuration: JSON policy configuration (must include 'start_ns').82 capacity: Most outstanding requests.83 startup_timeout: Seconds to wait for the child's 'ready'.84 clock: Shared clock; a new one starts at 'start_ns' if omitted.85 on_notice: Receives Runtime notices on the receiver thread.86 on_failure: Receives the first Runtime failure.87 """8889 # Set in __init__; declared here because the IO methods are defined first.90 clock: SharedExperimentClock91 _lock: RLock92 _ready: Event93 _stop: StopSignal94 _local_stop: Event95 _queue: Queue[bytes]96 _control_queue: Queue[bytes]97 _control_ids: Set[str]98 _pending: Dict[str, Future[Any]]99 _number: int100 _capacity: int101 _view: Dict[str, Any]102 _error: Optional[Exception]103 _closed: bool104 _callbacks_bound: bool105 _on_notice: NoticeCallback106 _on_failure: FailureCallback107 _write: Connection108 _control_write: Connection109 _read: Connection110 _client_outgoing: Connection111 _client_incoming: Connection112 _client_borrowed: bool113 _process: BaseProcess114 _sender: Thread115 _control_sender: Thread116 _receiver: Thread117118 def _fail(self, error: Exception, *, notify: bool = True) -> None:119 with self._lock:120 if self._error is not None:121 return122 self._error = error123 pending, self._pending = self._pending, {}124 self._local_stop.set()125 self._stop.set()126 self.clock.changed.release()127 for future in pending.values():128 if not future.done():129 future.set_exception(error)130 if notify:131 self._on_failure(error)132133 def _send_control(self) -> None:134 self._send(self._control_queue, self._control_write)135136 def _send(137 self, queue: Optional[Queue[bytes]] = None, outgoing: Optional[Connection] = None138 ) -> None:139 queue = self._queue if queue is None else queue140 outgoing = self._write if outgoing is None else outgoing141 try:142 while not self._local_stop.is_set():143 try:144 raw = queue.get(timeout=SEND_POLL_SECONDS)145 except Empty:146 continue147 outgoing.send_bytes(raw)148 except (OSError, EOFError):149 if not self._stop.is_set():150 self._fail(ContractError('runtime request channel failed'))151152 def _message(self, message: Message) -> None:153 reply(message)154 kind = message.get('kind')155 if kind == 'ready':156 self._ready.set()157 return158 if kind == 'notice':159 self._on_notice(message['event'], copy_json(message['details']))160 return161 if kind not in ('snapshot', 'reply', 'ended'):162 raise ContractError('invalid runtime reply kind')163 with self._lock:164 self._view = copy_json(message['snapshot'])165 future = self._pending.pop(message['id'], None) if kind == 'reply' else None166 if kind == 'reply':167 self._control_ids.discard(message['id'])168 if kind == 'reply':169 if future is None:170 raise ContractError('unexpected runtime reply identity')171 if message['error'] is not None:172 error = message['error']173 future.set_exception(RuntimeError(error['code'], error['message']))174 else:175 future.set_result(copy_json(message['result']))176177 def _receive(self) -> None:178 try:179 while True:180 self._message(unpack(self._read.recv_bytes(MAX_BYTES)))181 except (OSError, EOFError):182 self._fail(183 ContractError(184 'runtime stopped' if self._stop.is_set() else 'runtime process disconnected'185 ),186 notify=not self._stop.is_set(),187 )188 except Exception as exc:189 _log.warning('Handled failure in _receive', exc_info=True)190 self._fail(exc)191192 def request_stop(self) -> None:193 """Reserved shared stop signal, independent of a full command pipe."""194 self._stop.set()195 self._local_stop.set()196 self.clock.changed.release()197198 def close(self) -> None:199 if self._closed:200 return201 self.request_stop()202 if self._process.pid is not None:203 self._process.join(PROCESS_JOIN_SECONDS)204 if self._process.is_alive():205 self._process.terminate()206 self._process.join(PROCESS_JOIN_SECONDS)207 if self._process.is_alive():208 self._process.kill()209 self._process.join(PROCESS_JOIN_SECONDS)210 if self._process.is_alive():211 raise ContractError('runtime process did not close; ownership retained')212 for thread in (self._sender, self._control_sender, self._receiver):213 if thread.ident is not None:214 thread.join(THREAD_JOIN_SECONDS)215 if thread.is_alive():216 raise ContractError('runtime IO did not close; ownership retained')217 self._write.close()218 self._control_write.close()219 self._read.close()220 self._client_outgoing.close()221 self._client_incoming.close()222 self._closed = True223224 def __init__(225 self,226 factory: PolicyFactory,227 configuration: Dict[str, Any],228 *,229 capacity: int = DEFAULT_CAPACITY,230 startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,231 clock: Optional[SharedExperimentClock] = None,232 on_notice: Optional[NoticeCallback] = None,233 on_failure: Optional[FailureCallback] = None,234 ) -> None:235 integer(capacity, 'runtime capacity', 1)236 if capacity > MAX_CAPACITY:237 raise ContractError('runtime capacity exceeds supported bound')238 if not callable(factory):239 raise ContractError('runtime requires a trusted policy factory')240 if (241 type(startup_timeout) not in (int, float)242 or not isfinite(startup_timeout)243 or startup_timeout <= 0244 or startup_timeout > MAX_STARTUP_TIMEOUT_SECONDS245 ):246 raise ContractError('invalid runtime startup timeout')247 configuration = copy_json(configuration)248 packet(configuration)249 if clock is not None and not isinstance(clock, SharedExperimentClock):250 raise ContractError('runtime requires a shared experiment clock')251 context = multiprocessing.get_context('spawn')252 self.clock = clock or SharedExperimentClock(context, configuration['start_ns'])253 self._ready, self._stop = Event(), StopSignal(context)254 self._local_stop = Event()255 self._queue = Queue(capacity)256 self._control_queue = Queue(MAX_PENDING_CONTROLS)257 self._control_ids = set()258 self._lock = RLock()259 self._pending = {}260 self._number = 0261 self._capacity = capacity262 self._view = {}263 self._error = None264 self._closed = False265 self._callbacks_bound = on_notice is not None or on_failure is not None266 self._on_notice = on_notice or (lambda event, details: None)267 self._on_failure = on_failure or (lambda error: None)268 child_in, self._write = context.Pipe(duplex=False)269 control_in, self._control_write = context.Pipe(duplex=False)270 self._read, child_out = context.Pipe(duplex=False)271 client_in, self._client_outgoing = context.Pipe(duplex=False)272 self._client_incoming, client_out = context.Pipe(duplex=False)273 self._client_borrowed = False274 self._process = context.Process(275 target=run_runtime,276 name='experiment-runtime',277 args=(278 child_in,279 child_out,280 client_in,281 client_out,282 control_in,283 self._stop,284 self.clock.reader(),285 factory,286 configuration,287 capacity,288 ),289 )290 self._sender = Thread(target=self._send, name='runtime-ipc-send', daemon=True)291 self._control_sender = Thread(292 target=self._send_control, name='runtime-control-send', daemon=True293 )294 self._receiver = Thread(target=self._receive, name='runtime-ipc-receive', daemon=True)295 try:296 self._process.start()297 child_in.close()298 control_in.close()299 child_out.close()300 client_in.close()301 client_out.close()302 self._sender.start()303 self._control_sender.start()304 self._receiver.start()305 deadline = time.monotonic() + startup_timeout306 while not self._ready.wait(READY_POLL_SECONDS):307 if not self._process.is_alive() or time.monotonic() >= deadline:308 raise ContractError('runtime process did not become ready')309 except Exception as exc:310 child_in.close()311 control_in.close()312 child_out.close()313 client_in.close()314 client_out.close()315 cleanup_after_error(exc, self.close)316 raise317318 def bind_callbacks(self, *, on_notice: NoticeCallback, on_failure: FailureCallback) -> None:319 """Bind lifecycle observers once during assembly, before Play.320321 Callbacks run on the Runtime receiver thread and must not wait for a322 Runtime reply. The handle retains transport/failure ownership.323 """324 with self._lock:325 if self._callbacks_bound or self._closed or self._pending:326 raise ContractError('bind Runtime observers once before use')327 self._on_notice = on_notice328 self._on_failure = on_failure329 self._callbacks_bound = True330331 def work_client(self) -> RuntimeClient:332 """Setup-only single borrowed endpoint; caller must own its carrier host."""333 with self._lock:334 if self._client_borrowed or self._closed or self._stop.is_set():335 raise ContractError('work capability already borrowed or runtime closed')336 self._client_borrowed = True337 return RuntimeClient(338 self._client_outgoing, self._client_incoming, self._stop, self.clock.reader()339 )340341 def submit(self, name: str, data: Optional[Dict[str, Any]] = None) -> Future[Any]:342 """Trusted adapter admission; never wait for child, clock or market progress."""343 data = copy_json({} if data is None else data)344 validate_payload(name, data)345 with self._lock:346 if self._closed or self._stop.is_set() or self._error is not None:347 raise ContractError('runtime is closed to new requests')348 control = name == 'control'349 if (control and len(self._control_ids) >= MAX_PENDING_CONTROLS) or (350 not control and len(self._pending) - len(self._control_ids) >= self._capacity351 ):352 raise RuntimeError('QUEUE_FULL', 'runtime capacity is full')353 self._number += 1354 identity = str(self._number)355 raw = packet(dict(id=identity, command=name, data=data))356 future: Future[Any] = Future()357 future.set_running_or_notify_cancel()358 self._pending[identity] = future359 try:360 if control:361 self._control_ids.add(identity)362 (self._control_queue if control else self._queue).put_nowait(raw)363 except Exception:364 del self._pending[identity]365 self._control_ids.discard(identity)366 raise367 return future368369 def snapshot(self) -> Dict[str, Any]:370 with self._lock:371 return dict(372 copy_json(self._view),373 pid=self._process.pid,374 alive=self._process.is_alive(),375 closed=self._closed,376 pending=len(self._pending),377 error=str(self._error) if self._error else None,378 )379380 def __enter__(self) -> ExperimentRuntimeHandle:381 return self382383 def __exit__(self, *_: object) -> None:384 self.close()