delivery.py
1"""Bounded ordered result/JSON delivery off the financial owner.23Slots are held from admission until delivery, including deferred Tools results.4A reserved terminal slot allows failure shutdown when ordinary capacity is full.5"""67from __future__ import annotations8import logging910from concurrent.futures import Future, CancelledError11from queue import Queue, Full12from threading import BoundedSemaphore, Event, RLock, Thread13from typing import Any, Callable, Dict, Optional, Tuple1415from sim_platform.domain.models import ContractError16from sim_platform.domain.json_values import copy_json17from sim_platform.experiments.execution.results import DeferredResult1819_log = logging.getLogger(__name__)2021# Notices queued at once when no capacity is given.22DEFAULT_EVENT_CAPACITY = 12823# Extra notice slots beyond the event capacity, for lifecycle notices.24NOTICE_RESERVE = 1625# The output queue's reserved slot for the terminal 'end' message.26TERMINAL_SLOTS = 12728# An output message: 'kind' ('result', 'notice', 'control' or 'end'), 'position' and29# kind-specific values; a 'result' may carry a DeferredResult until it is delivered.30OutputMessage = Dict[str, Any]31# Receives each delivered message (a detached copy).32Sink = Callable[[OutputMessage], None]333435def running_future() -> Future[Any]:36 """A Future already marked running, so it can no longer be cancelled."""37 result: Future[Any] = Future()38 result.set_running_or_notify_cancel()39 return result404142class ResultDelivery:43 """The output thread that publishes results and notices in order, off the owner.4445 The ExperimentSession creates one; it starts its output thread at once and the46 thread ends after delivering the 'end' message. ``slots`` bounds admitted47 requests until their result is delivered.4849 Args:50 capacity: Most admitted requests awaiting delivery.51 sink: Receives each delivered message.52 wake: Set when output fails, to wake the execution owner.53 event_capacity: Most queued notices.54 """5556 def __init__(57 self,58 capacity: int,59 sink: Sink,60 wake: Event,61 event_capacity: int = DEFAULT_EVENT_CAPACITY,62 ) -> None:63 self.sink, self.wake = sink, wake64 self.slots = BoundedSemaphore(capacity)65 self._notices = BoundedSemaphore(event_capacity + NOTICE_RESERVE)66 self.output: Queue[Tuple[OutputMessage, Optional[Future[Any]]]] = Queue(67 capacity + event_capacity + NOTICE_RESERVE + TERMINAL_SLOTS68 )69 self.delivery, self.cancel = Event(), Event()70 self.delivery.set()71 self.ended = running_future()72 self.output_error: Optional[BaseException] = None73 self._publication_lock = RLock()74 self._used = False75 self.output_thread = Thread(target=self._run, name='experiment-output', daemon=True)76 self.output_thread.start()7778 def reserve(self) -> None:79 """Take a result slot for a new request, or raise if closed or full."""80 if self.cancel.is_set() or not self.slots.acquire(blocking=False):81 raise ContractError('experiment closed or result capacity exhausted')82 self._used = True8384 def release(self) -> None:85 self.slots.release()8687 def bind_sink(self, sink: Sink) -> None:88 if self._used or self.cancel.is_set():89 raise ContractError('result delivery already used')90 self.sink = sink9192 def pause_delivery(self) -> None:93 with self._publication_lock:94 self.delivery.clear()9596 def send_output(self, message: OutputMessage, future: Optional[Future[Any]] = None) -> None:97 """Queue a message without blocking; ``future`` is resolved when it is delivered."""98 if message['kind'] == 'notice' and not self._notices.acquire(blocking=False):99 raise ContractError('experiment notification capacity exhausted')100 try:101 self.output.put_nowait((message, future))102 except Full:103 if message['kind'] == 'notice':104 self._notices.release()105 raise ContractError('experiment output capacity exhausted') from None106107 def _publish(self, message: OutputMessage, future: Optional[Future[Any]]) -> None:108 if self.output_error is not None:109 raise ContractError('experiment output failed') from self.output_error110 self.sink(copy_json(message))111 if future is not None and not future.done():112 future.set_result(copy_json(message))113114 def _run(self) -> None:115 while True:116 message, future = self.output.get()117 try:118 if isinstance(message.get('result'), DeferredResult):119 message = dict(message, result=message['result'].resolve())120 if message['kind'] == 'result':121 while True:122 self.delivery.wait()123 with self._publication_lock:124 if self.delivery.is_set():125 self._publish(message, future)126 break127 else:128 self._publish(message, future)129 except CancelledError:130 if future is not None and not future.done():131 future.set_exception(ContractError('experiment stopped before result'))132 except Exception as exc:133 _log.warning('Handled failure in _run', exc_info=True)134 self.output_error = exc135 if future is not None and not future.done():136 future.set_exception(exc)137 self.wake.set()138 finally:139 if message['kind'] == 'notice':140 self._notices.release()141 if future is not None and message['kind'] == 'result':142 self.slots.release()143 if message['kind'] == 'end':144 return