controls.py
1"""Prepared run barriers with acknowledgements from the actual capability owners.23Runtime and Execution confirm quiescence. Agent/Tools hosts confirm their4activation/delivery gates; this never claims arbitrary Python CPU code is paused.5"""67from __future__ import annotations89from threading import RLock10from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Protocol1112from sim_platform.domain.models import ContractError13from sim_platform.domain.json_values import copy_json14from sim_platform.resources import cleanup_after_error1516if TYPE_CHECKING: # Annotation only.17 from sim_platform.experiments.execution.session import ExperimentSession18 from sim_platform.experiments.scheduling.host import ExperimentRuntimeHandle1920# Internal coordination bounds, independent of user-configured model timeouts.21CONTROL_ACK_TIMEOUT_SECONDS = 522EXECUTION_DRAIN_TIMEOUT_SECONDS = 52324# A participant's or the Runtime's acknowledgement: 'participant' plus phase details.25Acknowledgement = Dict[str, Any]26# A confirmed control: 'request', 'state', 'acknowledgements' and 'position'.27ControlRecord = Dict[str, Any]282930class Participant(Protocol):31 """A host that confirms its own activation gate (the Tools host or a decision host).3233 play() and pause() return {'participant': 'tools' or 'decision', ...}.34 """3536 def play(self) -> Acknowledgement: ...3738 def pause(self) -> Acknowledgement: ...394041class DecisionParticipant(Participant, Protocol):42 """The Agent or Rule host, which can also be asked to stop."""4344 def request_stop(self) -> None: ...454647class ExperimentControls:48 """Play, Pause and Stop for one background experiment, acknowledged by each owner.4950 Assembly creates one per run and binds it to the runner, which calls it from the51 caller's thread. It borrows the pipeline, Runtime and hosts; stop() closes the52 pipeline and Runtime, and the runner closes the hosts.5354 Args:55 pipeline: The run's ExperimentSession.56 runtime: The run's Runtime process handle.57 decision: The Agent or Rule host, if any.58 tools: The Tools host, if any.59 record: Receives each confirmed control for the experiment log.60 """6162 def __init__(63 self,64 pipeline: ExperimentSession,65 runtime: ExperimentRuntimeHandle,66 *,67 decision: Optional[DecisionParticipant] = None,68 tools: Optional[Participant] = None,69 record: Optional[Callable[[ControlRecord], Any]] = None,70 ) -> None:71 self.pipeline, self.runtime = pipeline, runtime72 self.decision, self.tools = decision, tools73 self.record: Callable[[ControlRecord], Any] = record or (lambda value: None)74 self._lock = RLock()75 self.version = 076 self.state = 'ready'77 self.acknowledgements: List[ControlRecord] = []7879 def _request(self, action: str) -> Dict[str, Any]:80 self.version += 181 return dict(control_id=str(self.version), control_version=self.version, action=action)8283 def _runtime(self, request: Dict[str, Any]) -> Acknowledgement:84 return self.runtime.submit('control', request).result(CONTROL_ACK_TIMEOUT_SECONDS)8586 def _confirmed(87 self, request: Dict[str, Any], state: str, acks: List[Acknowledgement]88 ) -> ControlRecord:89 value = dict(90 request=request,91 state=state,92 acknowledgements=acks,93 position=self.pipeline.snapshot()['position'],94 )95 self.state = state96 self.acknowledgements = [copy_json(value)]97 self.record(value)98 return value99100 def _gates(self, action: str) -> List[Acknowledgement]:101 """Call ``action`` ('play' or 'pause') on each host and check its acknowledgement."""102 acknowledgements = []103 for resource, name in ((self.tools, 'tools'), (self.decision, 'decision')):104 if resource is None:105 continue106 result = getattr(resource, action)()107 if not isinstance(result, dict) or result.get('participant') != name:108 raise ContractError('participant did not acknowledge its capability gate')109 acknowledgements.append(result)110 return acknowledgements111112 def _failed_control(self, error: Exception) -> Optional[Dict[str, Any]]:113 """Resolve a concurrent terminal result or retain the original control failure."""114 if self.pipeline.snapshot()['state'] in ('completed', 'stopped', 'failed', 'ending'):115 terminal = self.pipeline.channels.ended.result(CONTROL_ACK_TIMEOUT_SECONDS)116 self.state = 'terminal'117 return dict(state='terminal', participant='execution', terminal=terminal)118 self.state = 'control_failed'119 cleanup_after_error(error, lambda: self.pipeline.control('stop'))120 return None121122 def play(self) -> Optional[Dict[str, Any]]:123 """Start or resume; None if already running, a terminal result if the run ended."""124 with self._lock:125 if self.state == 'running':126 return None127 if self.state not in ('ready', 'paused'):128 raise ContractError('experiment cannot start/resume from this state')129 request = self._request('play' if self.state == 'ready' else 'resume')130 try:131 runtime = self._runtime(request)132 if self.state == 'ready':133 self.pipeline.start()134 execution = dict(135 participant='execution',136 phase='started',137 position=self.pipeline.snapshot()['position'],138 )139 else:140 execution = self.pipeline.control_and_wait('resume')141 gates = self._gates('play')142 return self._confirmed(request, 'running', [runtime, execution, *gates])143 except Exception as error:144 terminal = self._failed_control(error)145 if terminal is not None:146 return terminal147 raise148149 def pause(self) -> Optional[Dict[str, Any]]:150 """Pause; None if not running, a terminal result if the run ended meanwhile."""151 with self._lock:152 if self.state != 'running':153 return None154 request = self._request('pause')155 self.state = 'pausing'156 try:157 # Clock freeze + admission close linearize in the Sequencer.158 applied = self.pipeline.control('pause')159 self.pipeline.channels.pause_delivery()160 gates = self._gates('pause')161 runtime = self._runtime(request)162 execution = applied.result(CONTROL_ACK_TIMEOUT_SECONDS)163 return self._confirmed(request, 'paused', [runtime, execution, *gates])164 except Exception as error:165 terminal = self._failed_control(error)166 if terminal is not None:167 return terminal168 raise169170 def stop(self) -> None:171 with self._lock:172 if self.state in ('terminal', 'resources_closed'):173 return174 self.state = 'stopping'175 request = self._request('stop')176 if self.decision:177 self.decision.request_stop()178 self.pipeline.close()179 self.runtime.close()180 # Terminal stream is already sealed. Keep detached final acks only.181 record, self.record = self.record, lambda value: None182 try:183 self._confirmed(184 request,185 'terminal',186 [187 dict(188 participant='execution',189 phase='terminal',190 position=self.pipeline.snapshot()['position'],191 ),192 dict(193 participant='runtime',194 phase='closed',195 closed=self.runtime.snapshot()['closed'],196 ),197 ],198 )199 finally:200 self.record = record201202 def snapshot(self) -> Dict[str, Any]:203 return dict(204 state=self.state,205 control_version=self.version,206 acknowledgements=copy_json(self.acknowledgements),207 )208209 def resources_closed(self) -> None:210 self.state = 'resources_closed'