session.py
1"""Shared prepared execution lifecycle used by method-specific DecisionRunners.23The owner merges source events and admitted commands; it never reconstructs a4native book. Result formatting/JSON/IPC delivery occur on the output stage.5This class is internal infrastructure, not another public experiment root.6"""78from __future__ import annotations9import logging1011from concurrent.futures import Future12from queue import Queue, Empty13from threading import Event, RLock, Thread, get_ident14from time import perf_counter_ns15from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Protocol, Tuple, Union, cast1617from sim_platform.experiments.scheduling.clock import real_delay_ns18from sim_platform.domain.models import ContractError19from sim_platform.domain.json_values import copy_json20from sim_platform.experiments.execution.timeline.sequencer import Sequencer21from sim_platform.experiments.execution.timeline.runner import ExecutionTimeline22from sim_platform.experiments.execution.metrics import RuntimeMetrics23from sim_platform.domain.operation_position import OperationPosition24from sim_platform.experiments.execution.delivery import ResultDelivery, running_future25from .controls import CONTROL_ACK_TIMEOUT_SECONDS, EXECUTION_DRAIN_TIMEOUT_SECONDS2627_log = logging.getLogger(__name__)2829if TYPE_CHECKING: # Annotation only.30 from sim_platform.market.models import MarketEvent31 from sim_platform.market.prepared.session import PreparedSession32 from sim_platform.experiments.execution.delivery import Sink33 from sim_platform.experiments.execution.market import LocalMarketExecution34 from sim_platform.experiments.execution.timeline.runner import TimelineReader35 from sim_platform.experiments.execution.timeline.sequencer import Command, CommandResult36 from sim_platform.experiments.scheduling.clock import ExperimentClock, SharedExperimentClock3738 # The run clock: in-process for policy runs, shared with children for Agent runs.39 RunClock = Union[ExperimentClock, SharedExperimentClock]4041# Default bounds: admitted commands awaiting results, and allowed execution lag.42DEFAULT_COMMAND_CAPACITY = 6443DEFAULT_LAG_LIMIT_NS = 100_000_00044# Longest idle wait of the owner loop between turns.45MAX_IDLE_WAIT_SECONDS = 0.0546NS_PER_SECOND = 1e947# Fewest consecutive idle events worth advancing as one span.48MIN_IDLE_SPAN = 24950# Wire position: 'sequence', 'time_ns', 'boundary' and 'ordinal' as decimal text.51WirePosition = Dict[str, str]52# Performs one admitted command: operation(payload, position) returns its result.53Operation = Callable[[Any, WirePosition], Any]54# Captures a market read: read(request, position, receipt_id) returns its result.55ReadOperation = Callable[[Any, WirePosition, str], Any]565758class Lifecycle(Protocol):59 """The run's scheduling binding (RuntimeBinding in application): it is told about60 financial changes, owner turns and the end, and publishes through ``notice``."""6162 @property63 def notice(self) -> Callable[[str, Dict[str, Any]], None]:64 """Publishes a notice: notice(event, details); the session replaces it."""6566 @notice.setter67 def notice(self, value: Callable[[str, Dict[str, Any]], None]) -> None: ...6869 def observe(self, position: WirePosition) -> None: ...7071 def step(self, position: WirePosition) -> None: ...7273 def end(self, reason: str) -> None: ...747576# Execution lifecycle only. Work/attempt and participant acknowledgements have77# separate owners; their states must not overwrite this financial lifecycle.78EXECUTION_TRANSITIONS = {79 'ready': {'running', 'ending'},80 'running': {'pausing', 'ending'},81 'pausing': {'paused', 'ending'},82 'paused': {'running', 'ending'},83 'ending': {'completed', 'stopped', 'failed'},84 'completed': set(),85 'stopped': set(),86 'failed': set(),87}888990class ExperimentSession:91 """The background financial owner of one experiment run.9293 Assembly creates it ready and borrows it the driver, reader and clock; it creates94 its Sequencer, ExecutionTimeline and ResultDelivery. start() launches the95 'execution-owner' thread, the only thread that changes financial state. close()96 stops the run and waits for the owner and output threads; it keeps ownership97 (raises) if either does not stop in time.9899 Args:100 driver: Market execution port for the run's market and ExecutionSession.101 reader: Prepared source event index.102 clock: Experiment clock; paused until play().103 start_ns: Experiment start time.104 end_ns: Experiment end time (exclusive), within the prepared source.105 sink: Receives delivered output messages.106 operation: Performs admitted commands; may be rebound by bind_ready().107 capacity: Most admitted commands awaiting results.108 lag_limit_ns: Largest allowed gap between experiment time and execution.109 lag_window_ns: How long the lag may exceed the limit before the run fails.110 replay_seed: Published as 'prepared_run' when the run starts.111 terminal_evidence: Returns extra values for the 'prepared_final' notice.112 """113114 # Set in __init__; declared here because _transition is defined first.115 _state: str116 channels: ResultDelivery117118 def _transition(self, state: str) -> None:119 """Change lifecycle at an owner/control safe point; terminal is final."""120 if state == self._state:121 return122 if state not in EXECUTION_TRANSITIONS[self._state]:123 raise ContractError('invalid execution lifecycle transition')124 self._state = state125126 def __init__(127 self,128 driver: LocalMarketExecution,129 reader: TimelineReader,130 *,131 clock: RunClock,132 start_ns: int,133 end_ns: int,134 sink: Sink,135 operation: Operation,136 capacity: int = DEFAULT_COMMAND_CAPACITY,137 lag_limit_ns: int = DEFAULT_LAG_LIMIT_NS,138 lag_window_ns: int = 0,139 replay_seed: Optional[Dict[str, Any]] = None,140 terminal_evidence: Optional[Callable[[], Dict[str, Any]]] = None,141 ) -> None:142 if not start_ns < end_ns <= reader.manifest['last_ns'] + 1:143 raise ContractError('experiment exceeds prepared source coverage')144 self.driver, self.reader, self.clock = driver, reader, clock145 self.end_ns, self.operation = end_ns, operation146 self.lag_limit_ns, self.lag_window_ns = lag_limit_ns, lag_window_ns147 self._replay_seed = copy_json(replay_seed)148 self._terminal_evidence: Callable[[], Dict[str, Any]] = terminal_evidence or (dict)149 self._wake, self._lock = Event(), RLock()150 self._controls: Queue[Tuple[str, Optional[Future[Any]]]] = Queue(4)151 self._thread: Optional[Thread] = None152 self._current: Optional[Command] = None153 self._state = 'ready'154 self._error: Optional[str] = None155 self._cleanup_errors: List[str] = []156 self._ordinal = 0157 self._stream_sealed = False158 self._receipt_number = 0159 self._read_operation: Optional[ReadOperation] = None160 self._pending: Dict[str, Future[Any]] = {}161 self._on_close: Callable[[], None] = lambda: None162 self._cancel_queries: Callable[[], None] = lambda: None163 self.lifecycle: Optional[Lifecycle] = None164 self._account_version: Optional[int] = None165 self._lag_since: Optional[int] = None166 self.metrics = RuntimeMetrics(lag_limit_ns)167 self.sequencer = Sequencer(168 clock.target_ns, start_ns=start_ns, end_ns=end_ns, capacity=capacity169 )170 self.timeline = ExecutionTimeline(171 reader,172 self.sequencer,173 self._advance,174 self._apply,175 self._finish,176 start_ns=start_ns,177 advance_span=self._advance_span,178 after_market=lambda: self._observe_financial(self._position()),179 )180 self.channels = ResultDelivery(capacity, sink, self._wake)181182 def _position(183 self, time_ns: Optional[int] = None, ordinal: Optional[int] = None184 ) -> WirePosition:185 return OperationPosition(186 sequence=self.driver.sequence,187 time_ns=self.timeline.frontier_ns if time_ns is None else time_ns,188 boundary=self.timeline.cursor,189 ordinal=self._ordinal if ordinal is None else ordinal,190 ).to_wire()191192 @property193 def receipt_index(self) -> str:194 """Request ID of the command the owner is applying now."""195 if self._current is None:196 raise ContractError('no owner operation is active')197 return self._current.request_id198199 def _emit(200 self,201 kind: str,202 future: Optional[Future[Any]] = None,203 *,204 position: Optional[WirePosition] = None,205 **values: Any,206 ) -> None:207 with self._lock:208 self._ordinal += 1209 published = dict(position or self._position(), ordinal=str(self._ordinal))210 self.channels.send_output(dict(kind=kind, position=published, **values), future)211212 def bind_ready(213 self,214 *,215 operation: Operation,216 sink: Sink,217 cancel_queries: Optional[Callable[[], None]] = None,218 ) -> None:219 if self._state != 'ready' or self._pending:220 raise ContractError('bind experiment only before use')221 self.operation = operation222 self.channels.bind_sink(sink)223 if cancel_queries is not None:224 self._cancel_queries = cancel_queries225226 def bind_lifecycle(self, lifecycle: Lifecycle) -> None:227 if self.lifecycle is not None or self._state != 'ready':228 raise ContractError('lifecycle already bound or started')229 self.lifecycle = lifecycle230 lifecycle.notice = lambda event, details: self._emit('notice', event=event, details=details)231232 def publish_notice(self, event: str, details: Dict[str, Any]) -> bool:233 """Queue a notice; False once the output stream has ended."""234 with self._lock:235 if self._stream_sealed:236 return False237 self._emit('notice', event=event, details=details)238 return True239240 def on_close(self, callback: Callable[[], None]) -> None:241 self._on_close = callback242243 def require_owner(self) -> None:244 if (245 self._thread is None246 or get_ident() != self._thread.ident247 or self._state not in ('running', 'pausing')248 ):249 raise ContractError('operation requires the financial owner')250251 def submit(self, payload: Any) -> Future[Any]:252 """Admit a command for the owner; the Future resolves when its result is delivered."""253 payload = copy_json(payload)254 with self._lock:255 if self._state != 'running':256 raise ContractError('experiment is not accepting commands')257 self.channels.reserve()258 identity = None259 try:260 identity = str(self._receipt_number + 1)261 command = self.sequencer.admit(identity, payload)262 future = running_future()263 self._pending[identity] = future264 self._receipt_number += 1265 prefix = self.reader.prefix(command.time_ns)266 self.publish_notice(267 'command_admitted',268 dict(269 command_id=identity,270 admission_sequence=str(command.sequence),271 effective_ns=str(command.time_ns),272 prepared_prefix=str(prefix),273 source_sequence=str(self.reader.source_sequence(prefix)),274 payload=payload,275 ),276 )277 except Exception:278 if identity in self._pending:279 # Admission already took effect. Fail closed and let the280 # terminal drain release its slot exactly once.281 self.sequencer.stop('failed')282 self._wake.set()283 else:284 self.channels.release()285 raise286 self._wake.set()287 return future288289 def bind_market_reads(self, operation: ReadOperation) -> None:290 if self._state != 'ready' or self._read_operation is not None:291 raise ContractError('bind market reads only before experiment starts')292 self._read_operation = operation293294 def submit_read(self, request: Any) -> Future[Any]:295 """Capture immutable market data at receipt without acquiring the financial owner.296297 Only bounded projection is performed here; Tools process formats results.298 Account/state queries still use the ordered execution barrier.299 """300 with self._lock:301 if self._state != 'running' or self._read_operation is None:302 raise ContractError('market read capability is not active')303 self.channels.reserve()304 try:305 cutoff = self.clock.target_ns()306 if cutoff > self.reader.manifest['last_ns']:307 raise ContractError('query exceeds prepared source coverage')308 prefix = self.reader.prefix(cutoff)309 position = dict(310 sequence=str(self.reader.market_sequence(prefix)),311 time_ns=str(cutoff),312 boundary=str(prefix),313 ordinal='0',314 )315 # Register only after capture succeeds. Otherwise the failure path316 # releases a slot while leaving a pending future for terminal drain.317 self._receipt_number += 1318 identity = str(self._receipt_number)319 future = running_future()320 self._pending[identity] = future321 except Exception:322 self.channels.release()323 raise324 try:325 result = self._read_operation(request, position, identity)326 with self._lock:327 if identity in self._pending:328 self._emit(329 'result',330 future,331 position=position,332 request=request,333 result=result,334 receipt_target_ns=str(cutoff),335 )336 self._pending.pop(identity)337 except Exception as exc:338 _log.warning('Handled failure in submit_read', exc_info=True)339 with self._lock:340 if self._pending.pop(identity, None) is not None:341 self.channels.release()342 future.set_exception(exc)343 return future344345 def _advance(self, index: int, time_ns: int) -> MarketEvent:346 actual = self.driver.pump()[0]347 # Background experiments always run on a prepared source.348 market = cast('PreparedSession', self.driver.market)349 if market.merged_prefix != index + 1 or actual.time_ns != time_ns:350 raise ContractError('prepared cursor and execution timeline differ')351 self.metrics.frame(self.clock.target_ns(), actual.time_ns)352 return actual353354 def _advance_span(self, index: int, limit: int) -> int:355 """Advance a run of idle events at once when the reader and driver support it."""356 if not hasattr(self.reader, 'idle_span_end') or not hasattr(357 self.driver, 'advance_idle_prefix'358 ):359 return 0360 end = min(limit, self.reader.idle_span_end(index))361 if end - index < MIN_IDLE_SPAN:362 return 0363 count, event = self.driver.advance_idle_prefix(end)364 if count:365 self.metrics.span(self.clock.target_ns(), self.reader.event_time(index), count)366 return count367368 def _observe_financial(self, position: WirePosition) -> None:369 if self.lifecycle is None:370 return371 version = self.driver.account_sequence372 if version != self._account_version:373 self.lifecycle.observe(position)374 self._account_version = version375376 def _apply(self, command: Command) -> CommandResult:377 self._current = command378 started = perf_counter_ns()379 try:380 position = self._position(command.time_ns, self._ordinal + 1)381 account_before = getattr(self.driver, 'account_sequence', None)382 result = self.operation(command.payload, position)383 self._observe_financial(position)384 self.publish_notice(385 'command_processed',386 dict(387 command_id=command.request_id,388 effective_ns=str(command.time_ns),389 admission_sequence=str(command.sequence),390 financial_effect=(391 (getattr(self.driver, 'account_sequence', None) != account_before)392 if account_before is not None393 else None394 ),395 ),396 )397 future = self._pending[command.request_id]398 self._emit(399 'result',400 future,401 position=position,402 request=command.payload,403 result=result,404 receipt_target_ns=str(command.time_ns),405 )406 self._pending.pop(command.request_id)407 self.metrics.operation(perf_counter_ns() - started)408 return dict(status='processed')409 finally:410 self._current = None411412 def _finish(self, reason: str, stamp: int) -> None:413 status = 'completed' if reason == 'end' else 'failed' if reason == 'failed' else 'stopped'414 self._terminate(status, reason, stamp)415416 def _record_error(self, error: object) -> None:417 """Retain the initiating failure and separately expose later cleanup errors."""418 if self._error is None:419 self._error = str(error)420 else:421 self._cleanup_errors.append(str(error))422423 def _cancel_requests(self, reason: str) -> None:424 for identity, future in list(self._pending.items()):425 if future.done():426 continue427 try:428 self.publish_notice('command_cancelled', dict(command_id=identity, reason=reason))429 except Exception as exc:430 _log.warning('Handled failure in _cancel_requests', exc_info=True)431 self._record_error(exc)432 future.set_exception(ContractError('experiment ended before operation'))433 self.channels.release()434 self._pending.clear()435 while True:436 try:437 _, control_future = self._controls.get_nowait()438 except Empty:439 break440 if control_future is not None and not control_future.done():441 control_future.set_exception(442 ContractError('experiment ended before control acknowledgement')443 )444445 def _stop_participants(self, reason: str) -> None:446 try:447 self._cancel_queries()448 except Exception as exc:449 _log.warning('Handled failure in _stop_participants', exc_info=True)450 self._record_error(exc)451 if self.lifecycle is not None:452 try:453 self.lifecycle.end(reason)454 except Exception as exc:455 _log.warning('Handled failure in _stop_participants', exc_info=True)456 self._record_error(exc)457 self._cancel_requests(reason)458459 def _finish_financial(self, status: str, reason: str, stamp: int) -> None:460 try:461 self.driver.finish('failed' if self._error else status, reason, stamp)462 except Exception as exc:463 _log.warning('Handled failure in _finish_financial', exc_info=True)464 self._record_error(exc)465 try:466 self.clock.pause()467 except Exception as exc:468 _log.warning('Handled failure in _finish_financial', exc_info=True)469 self._record_error(exc)470 if hasattr(self.driver, 'execution'):471 try:472 from dataclasses import asdict473474 self.publish_notice(475 'prepared_final',476 dict(477 financial=asdict(self.driver.execution.snapshot),478 position=self._position(),479 **self._terminal_evidence(),480 ),481 )482 except Exception as exc:483 _log.warning('Handled failure in _finish_financial', exc_info=True)484 self._record_error(exc)485486 def _terminate(self, status: str, reason: str, stamp: Optional[int] = None) -> None:487 if self._state in ('completed', 'stopped', 'failed', 'ending'):488 return489 # Terminal state becomes visible only after the financial close attempt.490 # Output acknowledgement remains separate: channels.ended confirms drain.491 self._transition('ending')492 self._error = reason if status == 'failed' else None493 self.channels.cancel.set()494 self.channels.delivery.set()495 self._stop_participants(reason)496 self._finish_financial(497 status, reason, self.timeline.frontier_ns if stamp is None else stamp498 )499 with self._lock:500 self._transition('failed' if self._error else status)501 self._emit(502 'end',503 self.channels.ended,504 status=self._state,505 reason=self._error or reason,506 evaluation=self.metrics.snapshot(self._state),507 )508 self._stream_sealed = True509510 def play(self) -> None:511 if self._state != 'ready':512 raise ContractError('experiment already started')513 if self._replay_seed is not None:514 self.publish_notice('prepared_run', self._replay_seed)515 self.driver.play()516 self.clock.resume()517 self._transition('running')518519 def start(self) -> None:520 self.play()521 self._thread = Thread(target=self._run, name='execution-owner', daemon=True)522 self._thread.start()523524 def wake(self) -> None:525 self._wake.set()526527 def control(self, command: str) -> Future[Any]:528 """Queue 'pause', 'resume' or 'stop'; the Future resolves when the owner applies it."""529 if command not in ('pause', 'resume', 'stop'):530 raise ContractError('unknown experiment control')531 with self._lock:532 if self._state in ('completed', 'stopped', 'failed'):533 return self.channels.ended534 result: Optional[Future[Any]] = running_future()535 if command == 'stop':536 self.sequencer.stop()537 result = None538 elif command == 'pause':539 self.sequencer.pause(self.clock.pause)540 self._transition('pausing')541 self._controls.put_nowait((command, result))542 self._wake.set()543 # result is None only for 'stop', which returns the end Future instead.544 return self.channels.ended if command == 'stop' else cast('Future[Any]', result)545546 def control_and_wait(self, command: str, timeout: float = CONTROL_ACK_TIMEOUT_SECONDS) -> Any:547 return self.control(command).result(timeout)548549 def _service_control(self) -> None:550 try:551 command, future = self._controls.get_nowait()552 except Empty:553 return554 try:555 if command == 'stop':556 self.timeline.turn()557 elif command == 'pause':558 self.timeline.turn() # Atomically sealed <=pause cutoff, including same-tick commands.559 if self._state in ('completed', 'stopped', 'failed'):560 raise ContractError('experiment ended before pause acknowledgement')561 self.driver.pause()562 self._transition('paused')563 else:564 self.driver.play()565 self.sequencer.resume(self.clock.resume)566 self._transition('running')567 self.channels.delivery.set()568 result = dict(569 event=command,570 applied=True,571 participant='execution',572 phase='quiescent' if command == 'pause' else 'running',573 position=self._position(),574 state=self._state,575 )576 self._emit('control', event=command, applied=True)577 if future is not None:578 future.set_result(result)579 except Exception as exc:580 if future is not None:581 future.set_exception(exc)582 raise583584 def tick(self) -> Dict[str, Any]:585 """One owner turn: apply controls and due work; any error ends the run as failed."""586 self.metrics.turns += 1587 if self._thread is not None and get_ident() != self._thread.ident:588 raise ContractError('execution owner exclusively drives experiment')589 try:590 self._service_control()591 if self.channels.output_error is not None:592 raise ContractError('experiment result delivery failed')593 if self._state == 'running':594 self.timeline.turn()595 if self.lifecycle is not None:596 self.lifecycle.step(self._position())597 lag = min(self.clock.target_ns(), self.end_ns) - self.timeline.frontier_ns598 if lag > self.lag_limit_ns:599 if self._lag_since is None:600 self._lag_since = self.clock.target_ns()601 if self.clock.target_ns() - self._lag_since >= self.lag_window_ns:602 raise ContractError('execution frontier lag limit exceeded')603 else:604 self._lag_since = None605 except Exception as exc:606 _log.warning('Handled failure in tick', exc_info=True)607 self.sequencer.stop('failed')608 self._terminate('failed', str(exc))609 return self.snapshot()610611 def _run(self) -> None:612 while self._state not in ('completed', 'stopped', 'failed'):613 self._wake.clear()614 self.tick()615 if self._state in ('completed', 'stopped', 'failed'):616 break617 next_ns = (618 self.reader.event_time(self.timeline.cursor)619 if self.timeline.cursor < self.reader.manifest['frames']620 else self.end_ns621 )622 delay = min(623 MAX_IDLE_WAIT_SECONDS,624 real_delay_ns(self.clock, min(next_ns, self.end_ns)) / NS_PER_SECOND,625 )626 if self.sequencer.snapshot()['pending']:627 delay = 0628 self._wake.wait(delay if self._state == 'running' else None)629630 def snapshot(self) -> Dict[str, Any]:631 try:632 target = min(self.clock.target_ns(), self.end_ns)633 except ContractError:634 # A Runtime killed while holding the shared clock's lock leaves the clock635 # unreadable; the owner then ends the run as failed. Report the executed636 # position instead, so status reads and close() still work.637 _log.warning('Experiment clock unreadable; reporting the executed position')638 target = self.timeline.frontier_ns639 return dict(640 state=self._state,641 target_ns=target,642 executed_ns=self.timeline.frontier_ns,643 lag_ns=max(0, target - self.timeline.frontier_ns),644 position=self._position(),645 error=self._error,646 cleanup_errors=list(self._cleanup_errors),647 evaluation=self.metrics.snapshot(self._state),648 output_error=str(self.channels.output_error) if self.channels.output_error else None,649 )650651 @property652 def quiescent(self) -> bool:653 return (654 self._thread is None or not self._thread.is_alive()655 ) and not self.channels.output_thread.is_alive()656657 def close(self) -> None:658 if self._state not in ('completed', 'stopped', 'failed'):659 future = self.control('stop')660 if self._thread is None:661 self.tick()662 future.result(CONTROL_ACK_TIMEOUT_SECONDS)663 if self._thread is not None:664 self._thread.join(EXECUTION_DRAIN_TIMEOUT_SECONDS)665 if self._thread.is_alive():666 raise ContractError('execution owner did not stop; ownership retained')667 self.channels.output_thread.join(EXECUTION_DRAIN_TIMEOUT_SECONDS)668 if self.channels.output_thread.is_alive():669 raise ContractError('result delivery did not drain; ownership retained')670 self._on_close()671 if self.channels.output_error is not None:672 raise ContractError('experiment output failed') from self.channels.output_error