m/market labResearch, with a record.
Source on GitHub ↗

clock.py

1"""Parent-written effective clock, read by spawn children over one POSIX epoch.23Clock reads use a bounded process lock: child death while holding it faults the4clock instead of hanging the app. Stop/wake and close never acquire this lock.5The wake semaphore has no process-owned mutex to strand during termination.6"""78from __future__ import annotations910from bisect import bisect_right11import os12import sys13import time14from threading import RLock15from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Sequence1617from sim_platform.domain.models import ContractError, integer1819if TYPE_CHECKING:  # Annotation only.20    from multiprocessing.context import BaseContext21    from multiprocessing.synchronize import Lock, Semaphore2223# Longest wait for the shared clock lock; a timeout means a child died holding it.24LOCK_TIMEOUT_SECONDS = 0.12526# A clock reading: 'elapsed_ns', 'started_ns', 'running', 'version' and 'target_ns'.27ClockSnapshot = Dict[str, Any]282930# After the last event of a session the clock keeps running this long before it skips31# the closed market, so work already in progress can finish at its normal pace.32GAP_GRACE_NS = 5_000_000_00033# The clock lands this long before the next session's first event.34GAP_LEAD_NS = 2_000_000_00035# A turn-based clock never moves by itself; waiters re-read it at least this often.36PACED_POLL_NS = 20_000_000373839class SessionGaps:40    """Stretches of market time, such as nights and weekends, that playback skips.4142    A recording that covers several days has no events while the market is closed.43    Running through those hours in real time would be pointless, so the experiment44    clock jumps over them. This class holds the arithmetic for that jump.4546    Two time lines are involved. *Market time* is what the recording and every view47    use. *Running time* is market time with the skipped stretches removed: it advances48    exactly as fast as real time while the experiment plays. A clock stores running49    time and converts it to market time whenever it is read.5051    Args:52        closed: (start, end) market times between which the recording has no events.53        origin_ns: Where the experiment starts; stretches before it are irrelevant.54        grace_ns: How long after ``start`` the clock still runs normally.55        lead_ns: How long before ``end`` the clock resumes.56    """5758    def __init__(59        self,60        closed: Iterable[Sequence[int]] = (),61        origin_ns: int = 0,62        *,63        grace_ns: int = GAP_GRACE_NS,64        lead_ns: int = GAP_LEAD_NS,65    ) -> None:66        self._starts: List[int] = []67        self._ends: List[int] = []68        # Running time at which each skip happens, and market time skipped up to and69        # including it.70        self._running_starts: List[int] = []71        self._skipped: List[int] = [0]72        for start, end in sorted((int(a), int(b)) for a, b in closed):73            start, end = max(start + grace_ns, origin_ns), end - lead_ns74            if end <= start or (self._ends and start < self._ends[-1]):75                continue76            self._starts.append(start)77            self._ends.append(end)78            self._running_starts.append(start - self._skipped[-1])79            self._skipped.append(self._skipped[-1] + end - start)8081    def __bool__(self) -> bool:82        return bool(self._starts)8384    def market_ns(self, running_ns: int) -> int:85        """Convert running time to market time; at a skip the result is the far side."""86        return running_ns + self._skipped[bisect_right(self._running_starts, running_ns)]8788    def running_ns(self, market_ns: int) -> int:89        """Convert market time to running time; any time inside a skip maps to its start."""90        passed = bisect_right(self._ends, market_ns)91        if passed < len(self._starts) and market_ns > self._starts[passed]:92            return self._running_starts[passed]93        return market_ns - self._skipped[passed]949596NO_GAPS = SessionGaps()979899def session_gaps(100    manifest: Dict[str, Any], origin_ns: int, timeout_seconds: Any = None101) -> SessionGaps:102    """Build the skips for an experiment on a prepared recording.103104    Only a recording prepared from price bars declares closed-market stretches; any105    other recording gives a clock that never skips. ``timeout_seconds`` is the longest106    a piece of work may take: the clock keeps running that much longer after a session107    ends, so work started just before the close is not cut short by the skip.108    """109    closed = (manifest.get('calendar') or {}).get('gaps') or ()110    if not closed:111        return NO_GAPS112    grace = GAP_GRACE_NS + (int(float(timeout_seconds) * 1e9) if timeout_seconds else 0)113    return SessionGaps(closed, origin_ns, grace_ns=grace)114115116def real_delay_ns(clock: Any, until_ns: int) -> int:117    """Real nanoseconds until the clock reaches ``until_ns`` if it keeps running.118119    A plain difference of market times would count hours the clock is going to skip.120    """121    delay = getattr(clock, 'delay_ns', None)122    return delay(until_ns) if delay is not None else max(0, until_ns - clock.target_ns())123124125def monotonic_ns() -> int:126    """Read a monotonic clock that all processes of the experiment share.127128    POSIX: CLOCK_MONOTONIC. Windows: QueryPerformanceCounter (time.monotonic_ns), one129    machine-wide counter, so values from different processes compare directly.130    """131    if hasattr(time, 'CLOCK_MONOTONIC'):132        return time.clock_gettime_ns(time.CLOCK_MONOTONIC)133    if sys.platform == 'win32':134        return time.monotonic_ns()135    raise ContractError('shared runtime clock requires a system-wide monotonic clock')136137138class StopSignal:139    """One parent-written byte: termination cannot inherit a killed child's lock."""140141    def __init__(self, context: BaseContext) -> None:142        self._value = context.RawValue('b', 0)143144    def is_set(self) -> bool:145        return bool(self._value.value)146147    def set(self) -> None:148        self._value.value = 1149150151class ClockReader:152    """Read-only view of a SharedExperimentClock, safe to pass to spawned children.153154    Created by SharedExperimentClock.reader(); it borrows the clock's shared state,155    lock and wake semaphore and owns nothing.156157    Args:158        state: Shared ctypes array of four int64 values: elapsed, started, running, version.159            ``elapsed`` is running time (see SessionGaps), not market time.160        lock: Process lock guarding ``state``.161        changed: Semaphore released whenever the clock or Runtime inputs change.162        gaps: Closed-market stretches this clock skips.163        paced: A turn-based clock: market time moves only when the owner advances it164            (advance_to), never with real time. ``running`` still says whether the165            experiment is playing.166    """167168    def __init__(169        self,170        state: Any,171        lock: Lock,172        changed: Semaphore,173        gaps: SessionGaps = NO_GAPS,174        paced: bool = False,175    ) -> None:176        self._state, self._lock, self.changed = state, lock, changed177        self._gaps = gaps178        self.paced = paced179180    def _read(self) -> Any:181        if not self._lock.acquire(timeout=LOCK_TIMEOUT_SECONDS):182            raise ContractError('shared clock unavailable; possible runtime failure')183        try:184            return self._state[:]185        finally:186            self._lock.release()187188    def snapshot(self) -> ClockSnapshot:189        elapsed, started, running, version = self._read()190        return dict(191            elapsed_ns=elapsed,192            started_ns=started,193            running=bool(running),194            version=version,195            target_ns=self._gaps.market_ns(196                elapsed + (monotonic_ns() - started if running and not self.paced else 0)197            ),198        )199200    def target_ns(self) -> int:201        """Current experiment time in market time, with closed stretches skipped."""202        return self.snapshot()['target_ns']203204    def delay_ns(self, until_ns: int) -> int:205        """Real nanoseconds until this clock reaches ``until_ns`` if it keeps running.206207        Derived from target_ns(), the one reading every user of the clock shares, so a208        clock that supplies its own time source stays consistent. A turn-based clock only209        moves when advanced, so a waiter is told to look again shortly instead.210        """211        now = self._gaps.running_ns(self.target_ns())212        delay = max(0, self._gaps.running_ns(until_ns) - now)213        return min(delay, PACED_POLL_NS) if self.paced else delay214215216class SharedExperimentClock(ClockReader):217    """Experiment time shared with child processes; only the creating process changes it.218219    Assembly creates one per Agent experiment and passes reader() copies to the220    Runtime and Agent children. It starts paused at ``origin_ns``. ``gaps`` lists the221    closed-market stretches of a multi-day recording, which the clock skips. ``paced``222    makes it turn-based: time moves only through advance_to(), driven by a TurnPacer.223    """224225    def __init__(226        self,227        context: BaseContext,228        origin_ns: int,229        gaps: SessionGaps = NO_GAPS,230        *,231        paced: bool = False,232    ) -> None:233        integer(origin_ns, 'clock origin')234        super().__init__(235            context.RawArray('q', [gaps.running_ns(origin_ns), 0, 0, 0]),236            context.Lock(),237            context.Semaphore(0),238            gaps,239            paced,240        )241        self._writer = os.getpid()242243    def reader(self) -> ClockReader:244        return ClockReader(self._state, self._lock, self.changed, self._gaps, self.paced)245246    def advance_to(self, market_ns: int) -> None:247        """Move a turn-based clock forward to ``market_ns`` (never backwards) and wake waiters."""248        integer(market_ns, 'clock target')249        if not self.paced:250            raise ContractError('only a turn-based clock is advanced explicitly')251        if os.getpid() != self._writer:252            raise ContractError('only the clock owner may change experiment time')253        if not self._lock.acquire(timeout=LOCK_TIMEOUT_SECONDS):254            raise ContractError('shared clock unavailable; possible runtime failure')255        try:256            elapsed, started, running, version = self._state[:]257            target = self._gaps.running_ns(market_ns)258            if target <= elapsed:259                return260            self._state[:] = [target, started, running, version + 1]261        finally:262            self._lock.release()263        self.changed.release()264265    def _change(self, running: bool) -> None:266        if os.getpid() != self._writer:267            raise ContractError('only the clock owner may change experiment time')268        if not self._lock.acquire(timeout=LOCK_TIMEOUT_SECONDS):269            raise ContractError('shared clock unavailable; possible runtime failure')270        try:271            elapsed, started, current, version = self._state[:]272            if bool(current) == running:273                return274            now = monotonic_ns()275            self._state[:] = [276                elapsed + (now - started if current and not self.paced else 0),277                now if running else 0,278                int(running),279                version + 1,280            ]281        finally:282            self._lock.release()283        self.changed.release()284285    def resume(self) -> None:286        self._change(True)287288    def pause(self) -> None:289        self._change(False)290291292class ExperimentClock:293    """In-process experiment time for policy experiments (no child processes).294295    The execution pipeline creates one per run when no shared clock is supplied.296    It starts paused at ``origin_ns``; ``now`` supplies monotonic nanoseconds. ``gaps``297    lists the closed-market stretches of a multi-day recording, which the clock skips.298    """299300    def __init__(301        self,302        origin_ns: int,303        now: Callable[[], int] = monotonic_ns,304        gaps: SessionGaps = NO_GAPS,305    ) -> None:306        integer(origin_ns, 'clock origin')307        self._now = now308        self._lock = RLock()309        self._gaps = gaps310        # Running time (see SessionGaps); converted to market time when read.311        self._elapsed = gaps.running_ns(origin_ns)312        self._started: Optional[int] = None313        self._previous: Optional[int] = None314315    def _read(self) -> int:316        value = self._now()317        integer(value, 'monotonic time')318        if self._previous is not None and value < self._previous:319            raise ContractError('monotonic clock moved backwards')320        self._previous = value321        return value322323    def _running_ns(self) -> int:324        with self._lock:325            now = self._read()326            return self._elapsed + (0 if self._started is None else now - self._started)327328    def target_ns(self) -> int:329        """Current experiment time in market time, with closed stretches skipped."""330        return self._gaps.market_ns(self._running_ns())331332    def delay_ns(self, until_ns: int) -> int:333        """Real nanoseconds until this clock reaches ``until_ns`` if it keeps running.334335        Derived from target_ns(), the one reading every user of the clock shares, so a336        clock that supplies its own time source stays consistent.337        """338        now = self._gaps.running_ns(self.target_ns())339        return max(0, self._gaps.running_ns(until_ns) - now)340341    def resume(self) -> None:342        with self._lock:343            if self._started is None:344                self._started = self._read()345346    def pause(self) -> None:347        with self._lock:348            if self._started is not None:349                self._elapsed += self._read() - self._started350                self._started = None