m/market labResearch, with a record.
Source on GitHub ↗

host.py

1"""Owned Runtime child with bounded nonblocking admission and detached replies.23Only sender/receiver threads wait on pipes; submit never blocks a market owner.4close is an off-owner operation, retains ownership if cleanup is unconfirmed.5The handle is infrastructure, not a second user-facing experiment API.6"""78from __future__ import annotations9import logging1011from concurrent.futures import Future12import multiprocessing13from queue import Queue, Empty14from threading import Thread, RLock, Event15import time16from math import isfinite17from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Set1819from sim_platform.domain.json_values import copy_json20from sim_platform.domain.models import ContractError, integer21from sim_platform.resources import cleanup_after_error22from sim_platform.experiments.scheduling.clock import SharedExperimentClock, StopSignal23from sim_platform.experiments.scheduling.protocol import (24    MAX_BYTES,25    packet,26    unpack,27    validate_payload,28    reply,29)30from sim_platform.experiments.scheduling.worker import run_runtime31from sim_platform.experiments.scheduling.client import RuntimeClient3233_log = logging.getLogger(__name__)3435if TYPE_CHECKING:  # Annotation only.36    from multiprocessing.connection import Connection37    from multiprocessing.process import BaseProcess3839    from sim_platform.experiments.scheduling.protocol import Message40    from sim_platform.experiments.scheduling.worker import NoticeCallback, PolicyFactory4142# Admission bounds: default and largest request capacity, and queued controls.43DEFAULT_CAPACITY = 6444MAX_CAPACITY = 102445MAX_PENDING_CONTROLS = 446# Child startup: default and longest wait for 'ready', and the poll interval.47DEFAULT_STARTUP_TIMEOUT_SECONDS = 548MAX_STARTUP_TIMEOUT_SECONDS = 6049READY_POLL_SECONDS = 0.0150# Sender threads re-check the stop flag at this interval.51SEND_POLL_SECONDS = 0.0552# close(): wait per termination step for the child, and for each IO thread.53PROCESS_JOIN_SECONDS = 0.554THREAD_JOIN_SECONDS = 15556# Receives a Runtime failure.57FailureCallback = Callable[[Exception], None]585960class RuntimeError(ContractError):61    """A Runtime refusal with a stable ``code`` (for example 'QUEUE_FULL').6263    Note: this name shadows the built-in RuntimeError inside this module.64    """6566    def __init__(self, code: str, message: str) -> None:67        self.code = code68        super().__init__(message)697071class ExperimentRuntimeHandle:72    """The App side of one Runtime child process and its pipes.7374    Created during experiment assembly; it starts the child, its sender/receiver75    threads, and waits for 'ready'. submit() never blocks; replies arrive on the76    receiver thread. close() stops and joins everything and raises (keeping77    ownership) if the child or a thread does not end.7879    Args:80        factory: Builds the work policy inside the child.81        configuration: JSON policy configuration (must include 'start_ns').82        capacity: Most outstanding requests.83        startup_timeout: Seconds to wait for the child's 'ready'.84        clock: Shared clock; a new one starts at 'start_ns' if omitted.85        on_notice: Receives Runtime notices on the receiver thread.86        on_failure: Receives the first Runtime failure.87    """8889    # Set in __init__; declared here because the IO methods are defined first.90    clock: SharedExperimentClock91    _lock: RLock92    _ready: Event93    _stop: StopSignal94    _local_stop: Event95    _queue: Queue[bytes]96    _control_queue: Queue[bytes]97    _control_ids: Set[str]98    _pending: Dict[str, Future[Any]]99    _number: int100    _capacity: int101    _view: Dict[str, Any]102    _error: Optional[Exception]103    _closed: bool104    _callbacks_bound: bool105    _on_notice: NoticeCallback106    _on_failure: FailureCallback107    _write: Connection108    _control_write: Connection109    _read: Connection110    _client_outgoing: Connection111    _client_incoming: Connection112    _client_borrowed: bool113    _process: BaseProcess114    _sender: Thread115    _control_sender: Thread116    _receiver: Thread117118    def _fail(self, error: Exception, *, notify: bool = True) -> None:119        with self._lock:120            if self._error is not None:121                return122            self._error = error123            pending, self._pending = self._pending, {}124        self._local_stop.set()125        self._stop.set()126        self.clock.changed.release()127        for future in pending.values():128            if not future.done():129                future.set_exception(error)130        if notify:131            self._on_failure(error)132133    def _send_control(self) -> None:134        self._send(self._control_queue, self._control_write)135136    def _send(137        self, queue: Optional[Queue[bytes]] = None, outgoing: Optional[Connection] = None138    ) -> None:139        queue = self._queue if queue is None else queue140        outgoing = self._write if outgoing is None else outgoing141        try:142            while not self._local_stop.is_set():143                try:144                    raw = queue.get(timeout=SEND_POLL_SECONDS)145                except Empty:146                    continue147                outgoing.send_bytes(raw)148        except (OSError, EOFError):149            if not self._stop.is_set():150                self._fail(ContractError('runtime request channel failed'))151152    def _message(self, message: Message) -> None:153        reply(message)154        kind = message.get('kind')155        if kind == 'ready':156            self._ready.set()157            return158        if kind == 'notice':159            self._on_notice(message['event'], copy_json(message['details']))160            return161        if kind not in ('snapshot', 'reply', 'ended'):162            raise ContractError('invalid runtime reply kind')163        with self._lock:164            self._view = copy_json(message['snapshot'])165            future = self._pending.pop(message['id'], None) if kind == 'reply' else None166            if kind == 'reply':167                self._control_ids.discard(message['id'])168        if kind == 'reply':169            if future is None:170                raise ContractError('unexpected runtime reply identity')171            if message['error'] is not None:172                error = message['error']173                future.set_exception(RuntimeError(error['code'], error['message']))174            else:175                future.set_result(copy_json(message['result']))176177    def _receive(self) -> None:178        try:179            while True:180                self._message(unpack(self._read.recv_bytes(MAX_BYTES)))181        except (OSError, EOFError):182            self._fail(183                ContractError(184                    'runtime stopped' if self._stop.is_set() else 'runtime process disconnected'185                ),186                notify=not self._stop.is_set(),187            )188        except Exception as exc:189            _log.warning('Handled failure in _receive', exc_info=True)190            self._fail(exc)191192    def request_stop(self) -> None:193        """Reserved shared stop signal, independent of a full command pipe."""194        self._stop.set()195        self._local_stop.set()196        self.clock.changed.release()197198    def close(self) -> None:199        if self._closed:200            return201        self.request_stop()202        if self._process.pid is not None:203            self._process.join(PROCESS_JOIN_SECONDS)204            if self._process.is_alive():205                self._process.terminate()206                self._process.join(PROCESS_JOIN_SECONDS)207            if self._process.is_alive():208                self._process.kill()209                self._process.join(PROCESS_JOIN_SECONDS)210            if self._process.is_alive():211                raise ContractError('runtime process did not close; ownership retained')212        for thread in (self._sender, self._control_sender, self._receiver):213            if thread.ident is not None:214                thread.join(THREAD_JOIN_SECONDS)215                if thread.is_alive():216                    raise ContractError('runtime IO did not close; ownership retained')217        self._write.close()218        self._control_write.close()219        self._read.close()220        self._client_outgoing.close()221        self._client_incoming.close()222        self._closed = True223224    def __init__(225        self,226        factory: PolicyFactory,227        configuration: Dict[str, Any],228        *,229        capacity: int = DEFAULT_CAPACITY,230        startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,231        clock: Optional[SharedExperimentClock] = None,232        on_notice: Optional[NoticeCallback] = None,233        on_failure: Optional[FailureCallback] = None,234    ) -> None:235        integer(capacity, 'runtime capacity', 1)236        if capacity > MAX_CAPACITY:237            raise ContractError('runtime capacity exceeds supported bound')238        if not callable(factory):239            raise ContractError('runtime requires a trusted policy factory')240        if (241            type(startup_timeout) not in (int, float)242            or not isfinite(startup_timeout)243            or startup_timeout <= 0244            or startup_timeout > MAX_STARTUP_TIMEOUT_SECONDS245        ):246            raise ContractError('invalid runtime startup timeout')247        configuration = copy_json(configuration)248        packet(configuration)249        if clock is not None and not isinstance(clock, SharedExperimentClock):250            raise ContractError('runtime requires a shared experiment clock')251        context = multiprocessing.get_context('spawn')252        self.clock = clock or SharedExperimentClock(context, configuration['start_ns'])253        self._ready, self._stop = Event(), StopSignal(context)254        self._local_stop = Event()255        self._queue = Queue(capacity)256        self._control_queue = Queue(MAX_PENDING_CONTROLS)257        self._control_ids = set()258        self._lock = RLock()259        self._pending = {}260        self._number = 0261        self._capacity = capacity262        self._view = {}263        self._error = None264        self._closed = False265        self._callbacks_bound = on_notice is not None or on_failure is not None266        self._on_notice = on_notice or (lambda event, details: None)267        self._on_failure = on_failure or (lambda error: None)268        child_in, self._write = context.Pipe(duplex=False)269        control_in, self._control_write = context.Pipe(duplex=False)270        self._read, child_out = context.Pipe(duplex=False)271        client_in, self._client_outgoing = context.Pipe(duplex=False)272        self._client_incoming, client_out = context.Pipe(duplex=False)273        self._client_borrowed = False274        self._process = context.Process(275            target=run_runtime,276            name='experiment-runtime',277            args=(278                child_in,279                child_out,280                client_in,281                client_out,282                control_in,283                self._stop,284                self.clock.reader(),285                factory,286                configuration,287                capacity,288            ),289        )290        self._sender = Thread(target=self._send, name='runtime-ipc-send', daemon=True)291        self._control_sender = Thread(292            target=self._send_control, name='runtime-control-send', daemon=True293        )294        self._receiver = Thread(target=self._receive, name='runtime-ipc-receive', daemon=True)295        try:296            self._process.start()297            child_in.close()298            control_in.close()299            child_out.close()300            client_in.close()301            client_out.close()302            self._sender.start()303            self._control_sender.start()304            self._receiver.start()305            deadline = time.monotonic() + startup_timeout306            while not self._ready.wait(READY_POLL_SECONDS):307                if not self._process.is_alive() or time.monotonic() >= deadline:308                    raise ContractError('runtime process did not become ready')309        except Exception as exc:310            child_in.close()311            control_in.close()312            child_out.close()313            client_in.close()314            client_out.close()315            cleanup_after_error(exc, self.close)316            raise317318    def bind_callbacks(self, *, on_notice: NoticeCallback, on_failure: FailureCallback) -> None:319        """Bind lifecycle observers once during assembly, before Play.320321        Callbacks run on the Runtime receiver thread and must not wait for a322        Runtime reply. The handle retains transport/failure ownership.323        """324        with self._lock:325            if self._callbacks_bound or self._closed or self._pending:326                raise ContractError('bind Runtime observers once before use')327            self._on_notice = on_notice328            self._on_failure = on_failure329            self._callbacks_bound = True330331    def work_client(self) -> RuntimeClient:332        """Setup-only single borrowed endpoint; caller must own its carrier host."""333        with self._lock:334            if self._client_borrowed or self._closed or self._stop.is_set():335                raise ContractError('work capability already borrowed or runtime closed')336            self._client_borrowed = True337            return RuntimeClient(338                self._client_outgoing, self._client_incoming, self._stop, self.clock.reader()339            )340341    def submit(self, name: str, data: Optional[Dict[str, Any]] = None) -> Future[Any]:342        """Trusted adapter admission; never wait for child, clock or market progress."""343        data = copy_json({} if data is None else data)344        validate_payload(name, data)345        with self._lock:346            if self._closed or self._stop.is_set() or self._error is not None:347                raise ContractError('runtime is closed to new requests')348            control = name == 'control'349            if (control and len(self._control_ids) >= MAX_PENDING_CONTROLS) or (350                not control and len(self._pending) - len(self._control_ids) >= self._capacity351            ):352                raise RuntimeError('QUEUE_FULL', 'runtime capacity is full')353            self._number += 1354            identity = str(self._number)355            raw = packet(dict(id=identity, command=name, data=data))356            future: Future[Any] = Future()357            future.set_running_or_notify_cancel()358            self._pending[identity] = future359            try:360                if control:361                    self._control_ids.add(identity)362                (self._control_queue if control else self._queue).put_nowait(raw)363            except Exception:364                del self._pending[identity]365                self._control_ids.discard(identity)366                raise367            return future368369    def snapshot(self) -> Dict[str, Any]:370        with self._lock:371            return dict(372                copy_json(self._view),373                pid=self._process.pid,374                alive=self._process.is_alive(),375                closed=self._closed,376                pending=len(self._pending),377                error=str(self._error) if self._error else None,378            )379380    def __enter__(self) -> ExperimentRuntimeHandle:381        return self382383    def __exit__(self, *_: object) -> None:384        self.close()