m/market labResearch, with a record.
Source on GitHub ↗

session.py

1"""Shared prepared execution lifecycle used by method-specific DecisionRunners.23The owner merges source events and admitted commands; it never reconstructs a4native book. Result formatting/JSON/IPC delivery occur on the output stage.5This class is internal infrastructure, not another public experiment root.6"""78from __future__ import annotations9import logging1011from concurrent.futures import Future12from queue import Queue, Empty13from threading import Event, RLock, Thread, get_ident14from time import perf_counter_ns15from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Protocol, Tuple, Union, cast1617from sim_platform.experiments.scheduling.clock import real_delay_ns18from sim_platform.domain.models import ContractError19from sim_platform.domain.json_values import copy_json20from sim_platform.experiments.execution.timeline.sequencer import Sequencer21from sim_platform.experiments.execution.timeline.runner import ExecutionTimeline22from sim_platform.experiments.execution.metrics import RuntimeMetrics23from sim_platform.domain.operation_position import OperationPosition24from sim_platform.experiments.execution.delivery import ResultDelivery, running_future25from .controls import CONTROL_ACK_TIMEOUT_SECONDS, EXECUTION_DRAIN_TIMEOUT_SECONDS2627_log = logging.getLogger(__name__)2829if TYPE_CHECKING:  # Annotation only.30    from sim_platform.market.models import MarketEvent31    from sim_platform.market.prepared.session import PreparedSession32    from sim_platform.experiments.execution.delivery import Sink33    from sim_platform.experiments.execution.market import LocalMarketExecution34    from sim_platform.experiments.execution.timeline.runner import TimelineReader35    from sim_platform.experiments.execution.timeline.sequencer import Command, CommandResult36    from sim_platform.experiments.scheduling.clock import ExperimentClock, SharedExperimentClock3738    # The run clock: in-process for policy runs, shared with children for Agent runs.39    RunClock = Union[ExperimentClock, SharedExperimentClock]4041# Default bounds: admitted commands awaiting results, and allowed execution lag.42DEFAULT_COMMAND_CAPACITY = 6443DEFAULT_LAG_LIMIT_NS = 100_000_00044# Longest idle wait of the owner loop between turns.45MAX_IDLE_WAIT_SECONDS = 0.0546NS_PER_SECOND = 1e947# Fewest consecutive idle events worth advancing as one span.48MIN_IDLE_SPAN = 24950# Wire position: 'sequence', 'time_ns', 'boundary' and 'ordinal' as decimal text.51WirePosition = Dict[str, str]52# Performs one admitted command: operation(payload, position) returns its result.53Operation = Callable[[Any, WirePosition], Any]54# Captures a market read: read(request, position, receipt_id) returns its result.55ReadOperation = Callable[[Any, WirePosition, str], Any]565758class Lifecycle(Protocol):59    """The run's scheduling binding (RuntimeBinding in application): it is told about60    financial changes, owner turns and the end, and publishes through ``notice``."""6162    @property63    def notice(self) -> Callable[[str, Dict[str, Any]], None]:64        """Publishes a notice: notice(event, details); the session replaces it."""6566    @notice.setter67    def notice(self, value: Callable[[str, Dict[str, Any]], None]) -> None: ...6869    def observe(self, position: WirePosition) -> None: ...7071    def step(self, position: WirePosition) -> None: ...7273    def end(self, reason: str) -> None: ...747576# Execution lifecycle only. Work/attempt and participant acknowledgements have77# separate owners; their states must not overwrite this financial lifecycle.78EXECUTION_TRANSITIONS = {79    'ready': {'running', 'ending'},80    'running': {'pausing', 'ending'},81    'pausing': {'paused', 'ending'},82    'paused': {'running', 'ending'},83    'ending': {'completed', 'stopped', 'failed'},84    'completed': set(),85    'stopped': set(),86    'failed': set(),87}888990class ExperimentSession:91    """The background financial owner of one experiment run.9293    Assembly creates it ready and borrows it the driver, reader and clock; it creates94    its Sequencer, ExecutionTimeline and ResultDelivery. start() launches the95    'execution-owner' thread, the only thread that changes financial state. close()96    stops the run and waits for the owner and output threads; it keeps ownership97    (raises) if either does not stop in time.9899    Args:100        driver: Market execution port for the run's market and ExecutionSession.101        reader: Prepared source event index.102        clock: Experiment clock; paused until play().103        start_ns: Experiment start time.104        end_ns: Experiment end time (exclusive), within the prepared source.105        sink: Receives delivered output messages.106        operation: Performs admitted commands; may be rebound by bind_ready().107        capacity: Most admitted commands awaiting results.108        lag_limit_ns: Largest allowed gap between experiment time and execution.109        lag_window_ns: How long the lag may exceed the limit before the run fails.110        replay_seed: Published as 'prepared_run' when the run starts.111        terminal_evidence: Returns extra values for the 'prepared_final' notice.112    """113114    # Set in __init__; declared here because _transition is defined first.115    _state: str116    channels: ResultDelivery117118    def _transition(self, state: str) -> None:119        """Change lifecycle at an owner/control safe point; terminal is final."""120        if state == self._state:121            return122        if state not in EXECUTION_TRANSITIONS[self._state]:123            raise ContractError('invalid execution lifecycle transition')124        self._state = state125126    def __init__(127        self,128        driver: LocalMarketExecution,129        reader: TimelineReader,130        *,131        clock: RunClock,132        start_ns: int,133        end_ns: int,134        sink: Sink,135        operation: Operation,136        capacity: int = DEFAULT_COMMAND_CAPACITY,137        lag_limit_ns: int = DEFAULT_LAG_LIMIT_NS,138        lag_window_ns: int = 0,139        replay_seed: Optional[Dict[str, Any]] = None,140        terminal_evidence: Optional[Callable[[], Dict[str, Any]]] = None,141    ) -> None:142        if not start_ns < end_ns <= reader.manifest['last_ns'] + 1:143            raise ContractError('experiment exceeds prepared source coverage')144        self.driver, self.reader, self.clock = driver, reader, clock145        self.end_ns, self.operation = end_ns, operation146        self.lag_limit_ns, self.lag_window_ns = lag_limit_ns, lag_window_ns147        self._replay_seed = copy_json(replay_seed)148        self._terminal_evidence: Callable[[], Dict[str, Any]] = terminal_evidence or (dict)149        self._wake, self._lock = Event(), RLock()150        self._controls: Queue[Tuple[str, Optional[Future[Any]]]] = Queue(4)151        self._thread: Optional[Thread] = None152        self._current: Optional[Command] = None153        self._state = 'ready'154        self._error: Optional[str] = None155        self._cleanup_errors: List[str] = []156        self._ordinal = 0157        self._stream_sealed = False158        self._receipt_number = 0159        self._read_operation: Optional[ReadOperation] = None160        self._pending: Dict[str, Future[Any]] = {}161        self._on_close: Callable[[], None] = lambda: None162        self._cancel_queries: Callable[[], None] = lambda: None163        self.lifecycle: Optional[Lifecycle] = None164        self._account_version: Optional[int] = None165        self._lag_since: Optional[int] = None166        self.metrics = RuntimeMetrics(lag_limit_ns)167        self.sequencer = Sequencer(168            clock.target_ns, start_ns=start_ns, end_ns=end_ns, capacity=capacity169        )170        self.timeline = ExecutionTimeline(171            reader,172            self.sequencer,173            self._advance,174            self._apply,175            self._finish,176            start_ns=start_ns,177            advance_span=self._advance_span,178            after_market=lambda: self._observe_financial(self._position()),179        )180        self.channels = ResultDelivery(capacity, sink, self._wake)181182    def _position(183        self, time_ns: Optional[int] = None, ordinal: Optional[int] = None184    ) -> WirePosition:185        return OperationPosition(186            sequence=self.driver.sequence,187            time_ns=self.timeline.frontier_ns if time_ns is None else time_ns,188            boundary=self.timeline.cursor,189            ordinal=self._ordinal if ordinal is None else ordinal,190        ).to_wire()191192    @property193    def receipt_index(self) -> str:194        """Request ID of the command the owner is applying now."""195        if self._current is None:196            raise ContractError('no owner operation is active')197        return self._current.request_id198199    def _emit(200        self,201        kind: str,202        future: Optional[Future[Any]] = None,203        *,204        position: Optional[WirePosition] = None,205        **values: Any,206    ) -> None:207        with self._lock:208            self._ordinal += 1209            published = dict(position or self._position(), ordinal=str(self._ordinal))210            self.channels.send_output(dict(kind=kind, position=published, **values), future)211212    def bind_ready(213        self,214        *,215        operation: Operation,216        sink: Sink,217        cancel_queries: Optional[Callable[[], None]] = None,218    ) -> None:219        if self._state != 'ready' or self._pending:220            raise ContractError('bind experiment only before use')221        self.operation = operation222        self.channels.bind_sink(sink)223        if cancel_queries is not None:224            self._cancel_queries = cancel_queries225226    def bind_lifecycle(self, lifecycle: Lifecycle) -> None:227        if self.lifecycle is not None or self._state != 'ready':228            raise ContractError('lifecycle already bound or started')229        self.lifecycle = lifecycle230        lifecycle.notice = lambda event, details: self._emit('notice', event=event, details=details)231232    def publish_notice(self, event: str, details: Dict[str, Any]) -> bool:233        """Queue a notice; False once the output stream has ended."""234        with self._lock:235            if self._stream_sealed:236                return False237            self._emit('notice', event=event, details=details)238            return True239240    def on_close(self, callback: Callable[[], None]) -> None:241        self._on_close = callback242243    def require_owner(self) -> None:244        if (245            self._thread is None246            or get_ident() != self._thread.ident247            or self._state not in ('running', 'pausing')248        ):249            raise ContractError('operation requires the financial owner')250251    def submit(self, payload: Any) -> Future[Any]:252        """Admit a command for the owner; the Future resolves when its result is delivered."""253        payload = copy_json(payload)254        with self._lock:255            if self._state != 'running':256                raise ContractError('experiment is not accepting commands')257            self.channels.reserve()258            identity = None259            try:260                identity = str(self._receipt_number + 1)261                command = self.sequencer.admit(identity, payload)262                future = running_future()263                self._pending[identity] = future264                self._receipt_number += 1265                prefix = self.reader.prefix(command.time_ns)266                self.publish_notice(267                    'command_admitted',268                    dict(269                        command_id=identity,270                        admission_sequence=str(command.sequence),271                        effective_ns=str(command.time_ns),272                        prepared_prefix=str(prefix),273                        source_sequence=str(self.reader.source_sequence(prefix)),274                        payload=payload,275                    ),276                )277            except Exception:278                if identity in self._pending:279                    # Admission already took effect. Fail closed and let the280                    # terminal drain release its slot exactly once.281                    self.sequencer.stop('failed')282                    self._wake.set()283                else:284                    self.channels.release()285                raise286        self._wake.set()287        return future288289    def bind_market_reads(self, operation: ReadOperation) -> None:290        if self._state != 'ready' or self._read_operation is not None:291            raise ContractError('bind market reads only before experiment starts')292        self._read_operation = operation293294    def submit_read(self, request: Any) -> Future[Any]:295        """Capture immutable market data at receipt without acquiring the financial owner.296297        Only bounded projection is performed here; Tools process formats results.298        Account/state queries still use the ordered execution barrier.299        """300        with self._lock:301            if self._state != 'running' or self._read_operation is None:302                raise ContractError('market read capability is not active')303            self.channels.reserve()304            try:305                cutoff = self.clock.target_ns()306                if cutoff > self.reader.manifest['last_ns']:307                    raise ContractError('query exceeds prepared source coverage')308                prefix = self.reader.prefix(cutoff)309                position = dict(310                    sequence=str(self.reader.market_sequence(prefix)),311                    time_ns=str(cutoff),312                    boundary=str(prefix),313                    ordinal='0',314                )315                # Register only after capture succeeds. Otherwise the failure path316                # releases a slot while leaving a pending future for terminal drain.317                self._receipt_number += 1318                identity = str(self._receipt_number)319                future = running_future()320                self._pending[identity] = future321            except Exception:322                self.channels.release()323                raise324        try:325            result = self._read_operation(request, position, identity)326            with self._lock:327                if identity in self._pending:328                    self._emit(329                        'result',330                        future,331                        position=position,332                        request=request,333                        result=result,334                        receipt_target_ns=str(cutoff),335                    )336                    self._pending.pop(identity)337        except Exception as exc:338            _log.warning('Handled failure in submit_read', exc_info=True)339            with self._lock:340                if self._pending.pop(identity, None) is not None:341                    self.channels.release()342                    future.set_exception(exc)343        return future344345    def _advance(self, index: int, time_ns: int) -> MarketEvent:346        actual = self.driver.pump()[0]347        # Background experiments always run on a prepared source.348        market = cast('PreparedSession', self.driver.market)349        if market.merged_prefix != index + 1 or actual.time_ns != time_ns:350            raise ContractError('prepared cursor and execution timeline differ')351        self.metrics.frame(self.clock.target_ns(), actual.time_ns)352        return actual353354    def _advance_span(self, index: int, limit: int) -> int:355        """Advance a run of idle events at once when the reader and driver support it."""356        if not hasattr(self.reader, 'idle_span_end') or not hasattr(357            self.driver, 'advance_idle_prefix'358        ):359            return 0360        end = min(limit, self.reader.idle_span_end(index))361        if end - index < MIN_IDLE_SPAN:362            return 0363        count, event = self.driver.advance_idle_prefix(end)364        if count:365            self.metrics.span(self.clock.target_ns(), self.reader.event_time(index), count)366        return count367368    def _observe_financial(self, position: WirePosition) -> None:369        if self.lifecycle is None:370            return371        version = self.driver.account_sequence372        if version != self._account_version:373            self.lifecycle.observe(position)374            self._account_version = version375376    def _apply(self, command: Command) -> CommandResult:377        self._current = command378        started = perf_counter_ns()379        try:380            position = self._position(command.time_ns, self._ordinal + 1)381            account_before = getattr(self.driver, 'account_sequence', None)382            result = self.operation(command.payload, position)383            self._observe_financial(position)384            self.publish_notice(385                'command_processed',386                dict(387                    command_id=command.request_id,388                    effective_ns=str(command.time_ns),389                    admission_sequence=str(command.sequence),390                    financial_effect=(391                        (getattr(self.driver, 'account_sequence', None) != account_before)392                        if account_before is not None393                        else None394                    ),395                ),396            )397            future = self._pending[command.request_id]398            self._emit(399                'result',400                future,401                position=position,402                request=command.payload,403                result=result,404                receipt_target_ns=str(command.time_ns),405            )406            self._pending.pop(command.request_id)407            self.metrics.operation(perf_counter_ns() - started)408            return dict(status='processed')409        finally:410            self._current = None411412    def _finish(self, reason: str, stamp: int) -> None:413        status = 'completed' if reason == 'end' else 'failed' if reason == 'failed' else 'stopped'414        self._terminate(status, reason, stamp)415416    def _record_error(self, error: object) -> None:417        """Retain the initiating failure and separately expose later cleanup errors."""418        if self._error is None:419            self._error = str(error)420        else:421            self._cleanup_errors.append(str(error))422423    def _cancel_requests(self, reason: str) -> None:424        for identity, future in list(self._pending.items()):425            if future.done():426                continue427            try:428                self.publish_notice('command_cancelled', dict(command_id=identity, reason=reason))429            except Exception as exc:430                _log.warning('Handled failure in _cancel_requests', exc_info=True)431                self._record_error(exc)432            future.set_exception(ContractError('experiment ended before operation'))433            self.channels.release()434        self._pending.clear()435        while True:436            try:437                _, control_future = self._controls.get_nowait()438            except Empty:439                break440            if control_future is not None and not control_future.done():441                control_future.set_exception(442                    ContractError('experiment ended before control acknowledgement')443                )444445    def _stop_participants(self, reason: str) -> None:446        try:447            self._cancel_queries()448        except Exception as exc:449            _log.warning('Handled failure in _stop_participants', exc_info=True)450            self._record_error(exc)451        if self.lifecycle is not None:452            try:453                self.lifecycle.end(reason)454            except Exception as exc:455                _log.warning('Handled failure in _stop_participants', exc_info=True)456                self._record_error(exc)457        self._cancel_requests(reason)458459    def _finish_financial(self, status: str, reason: str, stamp: int) -> None:460        try:461            self.driver.finish('failed' if self._error else status, reason, stamp)462        except Exception as exc:463            _log.warning('Handled failure in _finish_financial', exc_info=True)464            self._record_error(exc)465        try:466            self.clock.pause()467        except Exception as exc:468            _log.warning('Handled failure in _finish_financial', exc_info=True)469            self._record_error(exc)470        if hasattr(self.driver, 'execution'):471            try:472                from dataclasses import asdict473474                self.publish_notice(475                    'prepared_final',476                    dict(477                        financial=asdict(self.driver.execution.snapshot),478                        position=self._position(),479                        **self._terminal_evidence(),480                    ),481                )482            except Exception as exc:483                _log.warning('Handled failure in _finish_financial', exc_info=True)484                self._record_error(exc)485486    def _terminate(self, status: str, reason: str, stamp: Optional[int] = None) -> None:487        if self._state in ('completed', 'stopped', 'failed', 'ending'):488            return489        # Terminal state becomes visible only after the financial close attempt.490        # Output acknowledgement remains separate: channels.ended confirms drain.491        self._transition('ending')492        self._error = reason if status == 'failed' else None493        self.channels.cancel.set()494        self.channels.delivery.set()495        self._stop_participants(reason)496        self._finish_financial(497            status, reason, self.timeline.frontier_ns if stamp is None else stamp498        )499        with self._lock:500            self._transition('failed' if self._error else status)501            self._emit(502                'end',503                self.channels.ended,504                status=self._state,505                reason=self._error or reason,506                evaluation=self.metrics.snapshot(self._state),507            )508            self._stream_sealed = True509510    def play(self) -> None:511        if self._state != 'ready':512            raise ContractError('experiment already started')513        if self._replay_seed is not None:514            self.publish_notice('prepared_run', self._replay_seed)515        self.driver.play()516        self.clock.resume()517        self._transition('running')518519    def start(self) -> None:520        self.play()521        self._thread = Thread(target=self._run, name='execution-owner', daemon=True)522        self._thread.start()523524    def wake(self) -> None:525        self._wake.set()526527    def control(self, command: str) -> Future[Any]:528        """Queue 'pause', 'resume' or 'stop'; the Future resolves when the owner applies it."""529        if command not in ('pause', 'resume', 'stop'):530            raise ContractError('unknown experiment control')531        with self._lock:532            if self._state in ('completed', 'stopped', 'failed'):533                return self.channels.ended534            result: Optional[Future[Any]] = running_future()535            if command == 'stop':536                self.sequencer.stop()537                result = None538            elif command == 'pause':539                self.sequencer.pause(self.clock.pause)540                self._transition('pausing')541            self._controls.put_nowait((command, result))542        self._wake.set()543        # result is None only for 'stop', which returns the end Future instead.544        return self.channels.ended if command == 'stop' else cast('Future[Any]', result)545546    def control_and_wait(self, command: str, timeout: float = CONTROL_ACK_TIMEOUT_SECONDS) -> Any:547        return self.control(command).result(timeout)548549    def _service_control(self) -> None:550        try:551            command, future = self._controls.get_nowait()552        except Empty:553            return554        try:555            if command == 'stop':556                self.timeline.turn()557            elif command == 'pause':558                self.timeline.turn()  # Atomically sealed <=pause cutoff, including same-tick commands.559                if self._state in ('completed', 'stopped', 'failed'):560                    raise ContractError('experiment ended before pause acknowledgement')561                self.driver.pause()562                self._transition('paused')563            else:564                self.driver.play()565                self.sequencer.resume(self.clock.resume)566                self._transition('running')567                self.channels.delivery.set()568            result = dict(569                event=command,570                applied=True,571                participant='execution',572                phase='quiescent' if command == 'pause' else 'running',573                position=self._position(),574                state=self._state,575            )576            self._emit('control', event=command, applied=True)577            if future is not None:578                future.set_result(result)579        except Exception as exc:580            if future is not None:581                future.set_exception(exc)582            raise583584    def tick(self) -> Dict[str, Any]:585        """One owner turn: apply controls and due work; any error ends the run as failed."""586        self.metrics.turns += 1587        if self._thread is not None and get_ident() != self._thread.ident:588            raise ContractError('execution owner exclusively drives experiment')589        try:590            self._service_control()591            if self.channels.output_error is not None:592                raise ContractError('experiment result delivery failed')593            if self._state == 'running':594                self.timeline.turn()595                if self.lifecycle is not None:596                    self.lifecycle.step(self._position())597                lag = min(self.clock.target_ns(), self.end_ns) - self.timeline.frontier_ns598                if lag > self.lag_limit_ns:599                    if self._lag_since is None:600                        self._lag_since = self.clock.target_ns()601                    if self.clock.target_ns() - self._lag_since >= self.lag_window_ns:602                        raise ContractError('execution frontier lag limit exceeded')603                else:604                    self._lag_since = None605        except Exception as exc:606            _log.warning('Handled failure in tick', exc_info=True)607            self.sequencer.stop('failed')608            self._terminate('failed', str(exc))609        return self.snapshot()610611    def _run(self) -> None:612        while self._state not in ('completed', 'stopped', 'failed'):613            self._wake.clear()614            self.tick()615            if self._state in ('completed', 'stopped', 'failed'):616                break617            next_ns = (618                self.reader.event_time(self.timeline.cursor)619                if self.timeline.cursor < self.reader.manifest['frames']620                else self.end_ns621            )622            delay = min(623                MAX_IDLE_WAIT_SECONDS,624                real_delay_ns(self.clock, min(next_ns, self.end_ns)) / NS_PER_SECOND,625            )626            if self.sequencer.snapshot()['pending']:627                delay = 0628            self._wake.wait(delay if self._state == 'running' else None)629630    def snapshot(self) -> Dict[str, Any]:631        try:632            target = min(self.clock.target_ns(), self.end_ns)633        except ContractError:634            # A Runtime killed while holding the shared clock's lock leaves the clock635            # unreadable; the owner then ends the run as failed. Report the executed636            # position instead, so status reads and close() still work.637            _log.warning('Experiment clock unreadable; reporting the executed position')638            target = self.timeline.frontier_ns639        return dict(640            state=self._state,641            target_ns=target,642            executed_ns=self.timeline.frontier_ns,643            lag_ns=max(0, target - self.timeline.frontier_ns),644            position=self._position(),645            error=self._error,646            cleanup_errors=list(self._cleanup_errors),647            evaluation=self.metrics.snapshot(self._state),648            output_error=str(self.channels.output_error) if self.channels.output_error else None,649        )650651    @property652    def quiescent(self) -> bool:653        return (654            self._thread is None or not self._thread.is_alive()655        ) and not self.channels.output_thread.is_alive()656657    def close(self) -> None:658        if self._state not in ('completed', 'stopped', 'failed'):659            future = self.control('stop')660            if self._thread is None:661                self.tick()662            future.result(CONTROL_ACK_TIMEOUT_SECONDS)663        if self._thread is not None:664            self._thread.join(EXECUTION_DRAIN_TIMEOUT_SECONDS)665            if self._thread.is_alive():666                raise ContractError('execution owner did not stop; ownership retained')667        self.channels.output_thread.join(EXECUTION_DRAIN_TIMEOUT_SECONDS)668        if self.channels.output_thread.is_alive():669            raise ContractError('result delivery did not drain; ownership retained')670        self._on_close()671        if self.channels.output_error is not None:672            raise ContractError('experiment output failed') from self.channels.output_error