worker.py
1"""Spawn entrypoint: policy runs independently of IO publication and market work."""23from __future__ import annotations4import logging56from concurrent.futures import Future7from queue import Queue, Full, Empty8from threading import Thread9from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, Optional1011from sim_platform.domain.models import ContractError12from sim_platform.experiments.scheduling.protocol import (13 MAX_BYTES,14 Message,15 packet,16 unpack,17 command,18)1920_log = logging.getLogger(__name__)2122if TYPE_CHECKING: # Annotation only.23 from multiprocessing.connection import Connection2425 from multiprocessing.synchronize import Semaphore2627 from sim_platform.experiments.scheduling.clock import ClockReader, StopSignal28 from sim_platform.experiments.scheduling.policy.controller import LiveWorkPolicy2930# Publishes a Runtime notice: notice(event, details).31NoticeCallback = Callable[[str, Dict[str, Any]], None]32# Builds the work policy in the child: factory(configuration, clock, notice).33PolicyFactory = Callable[[Dict[str, Any], 'ClockReader', NoticeCallback], 'LiveWorkPolicy']3435# Queue sizes: replies/notices per capacity unit plus lifecycle reserve; Agent replies.36OUTPUT_RESERVE = 1637CLIENT_OUTPUT_CAPACITY = 238# Most messages read per turn from the control, application and Agent channels.39CONTROL_BATCH = 440APPLICATION_BATCH = 3241AGENT_BATCH = 242# How long the stopping loop waits for each sender thread to flush.43SENDER_JOIN_SECONDS = 0.5444546def send_messages(outgoing: Connection, output: Queue[Optional[Message]]) -> None:47 """Send queued messages until a None sentinel, then close the pipe."""48 try:49 while True:50 message = output.get()51 if message is None:52 return53 outgoing.send_bytes(packet(message))54 finally:55 outgoing.close()565758class RuntimeLoop:59 """The Runtime child's policy and output queues; one per Runtime process.6061 run_runtime creates it, starts its two sender threads and drives the policy from62 its single loop thread. Replies to the Agent go on a separate small queue.63 """6465 def emit(self, **message: Any) -> None:66 try:67 self.output.put_nowait(dict(**message))68 except Full as exc:69 raise ContractError('runtime output capacity exhausted') from exc7071 def notice(self, event: str, details: Dict[str, Any]) -> None:72 self.emit(kind='notice', event=event, details=details)7374 def __init__(75 self,76 outgoing: Connection,77 client_outgoing: Connection,78 clock: ClockReader,79 factory: PolicyFactory,80 configuration: Dict[str, Any],81 capacity: int,82 ) -> None:83 self.output: Queue[Optional[Message]] = Queue(capacity * 2 + OUTPUT_RESERVE)84 self.client_output: Queue[Optional[Message]] = Queue(CLIENT_OUTPUT_CAPACITY)85 self.sender = Thread(86 target=send_messages, args=(outgoing, self.output), name='runtime-output', daemon=True87 )88 self.client_sender = Thread(89 target=send_messages,90 args=(client_outgoing, self.client_output),91 name='runtime-work-output',92 daemon=True,93 )94 self.policy = factory(configuration, clock, self.notice)95 self._last: Optional[Dict[str, Any]] = None9697 def reply(self, identity: str, source: Future[Any], client: bool = False) -> None:98 """Queue the reply for request ``identity`` from a finished Future."""99 error: Optional[Dict[str, Any]] = None100 result: Any = None101 if source.cancelled():102 error = dict(code='RUNTIME_STOPPED', message='runtime stopped before work delivery')103 else:104 try:105 result = source.result()106 except Exception as exc:107 _log.warning('Handled failure in reply', exc_info=True)108 error = dict(code=getattr(exc, 'code', 'CONTRACT_ERROR'), message=str(exc))109 message = dict(110 kind='reply', id=identity, result=result, error=error, snapshot=self.policy.snapshot()111 )112 try:113 (self.client_output if client else self.output).put_nowait(message)114 except Full as exc:115 raise ContractError('runtime reply capacity exhausted') from exc116117 def dispatch(self, message: Message, client: bool = False) -> None:118 """Run one command on the policy and queue its reply (now or when it resolves)."""119 item = command(message)120 result: Future[Any] = Future()121 try:122 # A just-resumed clock can become visible before the next loop123 # turn. Dispatch due work before validating an immediate request.124 if item['command'] != 'control':125 self.policy.step()126 if client and item['command'] not in (127 'next_work',128 'finish',129 'wait',130 'begin_attempt',131 'model_receipt',132 'inspect',133 'agent_failed',134 ):135 raise ContractError('work capability does not grant this command')136 value = self.policy.dispatch(item['command'], item['data'])137 if isinstance(value, Future):138 value.add_done_callback(lambda source: self.reply(item['id'], source, client))139 return140 result.set_result(value)141 except (ContractError, KeyError, TypeError, ValueError) as exc:142 result.set_exception(exc)143 self.reply(item['id'], result, client)144145 def publish(self) -> None:146 snapshot = self.policy.snapshot()147 # Target time changes continuously; publish policy transitions only.148 state = {key: value for key, value in snapshot.items() if key != 'target_ns'}149 if state != self._last:150 self._last = state151 self.emit(kind='snapshot', snapshot=snapshot)152153154class RuntimeIngress:155 """IO-only reader; timer loop waits on a shared wake event, never a pipe.156157 One per input pipe, owned by RuntimeInputs; its daemon thread reads until the158 pipe fails and then records the error.159 """160161 # Set in __init__; declared here because read() is defined first.162 incoming: Connection163 wake: Semaphore164 queue: Queue[Message]165 error: Optional[BaseException]166167 def read(self) -> None:168 try:169 while True:170 raw = self.incoming.recv_bytes(MAX_BYTES)171 # recv framing is bounded before decoding/allocating a JSON graph.172 self.queue.put_nowait(unpack(raw))173 self.wake.release()174 except Exception as exc:175 _log.warning('Handled failure in read', exc_info=True)176 self.error = exc177 self.wake.release()178179 def drain(self, limit: int, *, stop: Optional[StopSignal] = None) -> Iterator[Message]:180 """Take at most limit queued messages without waiting on the IO thread."""181 for _ in range(limit):182 if stop is not None and stop.is_set():183 return184 try:185 yield self.queue.get_nowait()186 except Empty:187 return188189 def __init__(self, incoming: Connection, wake: Semaphore, capacity: int) -> None:190 self.incoming, self.wake = incoming, wake191 self.queue = Queue(capacity)192 self.error = None193 self.thread = Thread(target=self.read, name='runtime-input', daemon=True)194195196class RuntimeInputs:197 """Separate control, application and Agent channels around the serial policy loop.198199 Controls have a reserved queue. Each turn services bounded batches in that200 order, then returns to the timer policy. An Agent disconnect is a policy201 notification; losing either App channel is a Runtime failure.202 """203204 def _service_controls(self, loop: RuntimeLoop) -> None:205 if self.control.error is not None:206 raise ContractError('runtime control ingress failed') from self.control.error207 for message in self.control.drain(CONTROL_BATCH):208 if message.get('command') != 'control':209 raise ContractError('non-control on reserved channel')210 loop.dispatch(message)211212 def _service_agent(self, loop: RuntimeLoop) -> None:213 if self.agent.error is not None:214 loop.policy.dispatch('agent_failed', dict(reason='work capability disconnected'))215 self.agent.error = None216 for message in self.agent.drain(AGENT_BATCH):217 loop.dispatch(message, client=True)218219 def service(self, loop: RuntimeLoop, stop: StopSignal) -> None:220 """Run one bounded input turn; policy timeouts are serviced by the caller next."""221 self._service_controls(loop)222 if self.application.error is not None:223 raise ContractError('runtime ingress failed') from self.application.error224 for message in self.application.drain(APPLICATION_BATCH, stop=stop):225 loop.dispatch(message)226 self._service_agent(loop)227228 @property229 def pending(self) -> bool:230 return any(not source.queue.empty() for source in self.sources)231232 def start(self) -> None:233 for source in self.sources:234 source.thread.start()235236 def __init__(237 self,238 incoming: Connection,239 client_incoming: Connection,240 control_incoming: Connection,241 wake: Semaphore,242 capacity: int,243 ) -> None:244 self.application = RuntimeIngress(incoming, wake, capacity)245 self.agent = RuntimeIngress(client_incoming, wake, AGENT_BATCH)246 self.control = RuntimeIngress(control_incoming, wake, CONTROL_BATCH)247 self.sources = (self.application, self.agent, self.control)248249250def run_runtime(251 incoming: Connection,252 outgoing: Connection,253 client_incoming: Connection,254 client_outgoing: Connection,255 control_incoming: Connection,256 stop: StopSignal,257 clock: ClockReader,258 factory: PolicyFactory,259 configuration: Dict[str, Any],260 capacity: int,261) -> None:262 """Runtime child entry point: serve inputs and step the policy until stopped."""263 loop = None264 try:265 loop = RuntimeLoop(outgoing, client_outgoing, clock, factory, configuration, capacity)266 inputs = RuntimeInputs(incoming, client_incoming, control_incoming, clock.changed, capacity)267 loop.sender.start()268 loop.client_sender.start()269 inputs.start()270 loop.publish()271 loop.emit(kind='ready')272 while not stop.is_set():273 inputs.service(loop, stop)274 loop.policy.step()275 loop.publish()276 if inputs.pending:277 clock.changed.release()278 if not stop.is_set():279 clock.changed.acquire(timeout=loop.policy.wait_seconds())280 loop.policy.end('runtime stopped')281 loop.publish()282 loop.emit(kind='ended', snapshot=loop.policy.snapshot())283 loop.output.put_nowait(None)284 loop.client_output.put_nowait(None)285 loop.sender.join(SENDER_JOIN_SECONDS)286 loop.client_sender.join(SENDER_JOIN_SECONDS)287 finally:288 incoming.close()289 client_incoming.close()290 control_incoming.close()291 if loop is None:292 outgoing.close()293 client_outgoing.close()