host.py
1"""Owned persistent agent process; trusted work injection only, no upload loader.23Construction prepares a spawn child before market Play. A supervisor alone4reaps/terminates it; stop/EOF signals never wait on IPC or run on market owner.5The trusted program injection point supplies decision behavior.6"""78from __future__ import annotations9import logging1011from math import isfinite12import multiprocessing13from threading import Thread, RLock14import time15import sys16from typing import TYPE_CHECKING, Any, Callable, Dict, Mapping, Optional, Set1718from sim_platform.domain.json_values import copy_json19from sim_platform.domain.models import ContractError20from sim_platform.resources import cleanup_after_error21from sim_platform.agents.hosting.bridge import ToolPipeBridge22from sim_platform.experiments.scheduling.signals import ProcessSignal232425from .worker import run_program, idle_program26from .processes import create_process_family27from .lifecycle import AgentStartupError, reap_process_family2829_log = logging.getLogger(__name__)3031if TYPE_CHECKING: # Annotation only.32 from concurrent.futures import Future33 from multiprocessing.connection import Connection34 from multiprocessing.process import BaseProcess3536 from sim_platform.experiments.scheduling.client import RuntimeClient37 from sim_platform.models.contracts import ModelProfile, ModelProvider38 from sim_platform.sdk.context import AgentFactory39 from .protocol import TransportLimits4041# Startup errors that may be reported to the caller as they are.42REPORTED_STARTUP_ERRORS = (43 'AGENT_DEPENDENCY_MISSING',44 'AGENT_ENTRYPOINT_INVALID',45 'AGENT_TOOL_UNAVAILABLE',46)47# Grants the built-in template needs before a model may be configured.48TEMPLATE_MODEL_GRANTS = frozenset(('model.request', 'control.finish'))49# Startup and supervisor poll intervals, and default timeouts (seconds).50STARTUP_POLL_SECONDS = 0.0151SUPERVISOR_POLL_SECONDS = 0.0252DEFAULT_STARTUP_TIMEOUT_SECONDS = 553DEFAULT_SHUTDOWN_GRACE_SECONDS = 0.554# close() waits this long plus three grace periods for the supervisor.55SUPERVISOR_JOIN_BASE_SECONDS = 35657# Called once when the child exits: on_exit(state, error_text_or_None).58ExitCallback = Callable[[str, Optional[str]], Any]596061class AgentProcessHost:62 """Runner-owned host borrowing only request_tool and detached configuration.6364 program is trusted platform/test code, never resolved from a user manifest.65 A configured definition uses the built-in template. A supplied trusted program66 can replace it completely. JSON configuration contains no credentials or future67 market source paths. The runner owns the host: construction starts the child and68 waits until it is initialized; a supervisor thread reaps it; close() releases the69 process family, pipes and source workspace.70 """7172 # Set in __init__; declared here because the lifecycle methods are defined first.73 _lock: RLock74 _close_lock: RLock75 _closed: bool76 _stop_requested: bool77 _state: str78 _error: Optional[str]79 _exit_reason: Optional[str]80 _exit_notified: bool81 _family_reaped: bool82 _exitcode: Optional[int]83 _cleanup_error: Optional[BaseException]84 _grace: float85 _stop: ProcessSignal86 _running: ProcessSignal87 _ready: ProcessSignal88 _family_ready: ProcessSignal89 _process: BaseProcess90 _family: Any91 _bridge: ToolPipeBridge92 _supervisor: Thread93 _on_exit: ExitCallback94 _model_setup_write: Optional[Connection]95 _source_workspace: Optional[Any]9697 def request_stop(self) -> None:98 """Request cooperative stop without blocking the experiment execution owner."""99 with self._lock:100 if self._closed:101 return102 self._stop_requested = True103 if self._state not in ('failed', 'completed', 'stopped'):104 self._state = 'stopping'105 self._stop.set()106 self._running.set()107108 def _await_family(self, deadline: float) -> None:109 while not self._family_ready.wait(STARTUP_POLL_SECONDS):110 if not self._process.is_alive():111 raise AgentStartupError('AGENT_PROCESS_OWNERSHIP_FAILED')112 if time.monotonic() >= deadline:113 raise AgentStartupError('AGENT_INITIALIZATION_TIMEOUT')114115 def _await_initialization(self, deadline: float, startup: Connection) -> None:116 while not self._ready.wait(STARTUP_POLL_SECONDS):117 if self._stop.is_set() or not self._process.is_alive():118 raise AgentStartupError('AGENT_INITIALIZATION_FAILED')119 if time.monotonic() >= deadline:120 raise AgentStartupError('AGENT_INITIALIZATION_TIMEOUT')121 if not startup.poll():122 raise AgentStartupError('AGENT_INITIALIZATION_FAILED')123 try:124 error = startup.recv()125 except (EOFError, OSError) as exc:126 raise AgentStartupError('AGENT_INITIALIZATION_FAILED') from exc127 if error is not None:128 allowed = REPORTED_STARTUP_ERRORS129 raise AgentStartupError(error if error in allowed else 'AGENT_INITIALIZATION_FAILED')130 with self._lock:131 if self._stop.is_set():132 raise AgentStartupError('AGENT_INITIALIZATION_FAILED')133 self._state = 'ready'134135 def _reap(self) -> None:136 if not self._family_reaped:137 reap_process_family(self._process, self._family, self._grace)138 self._family_reaped = True139140 def _record_exit(self, requested: bool, natural_exit: Optional[int]) -> None:141 with self._lock:142 self._exitcode = self._process.exitcode143 transport_error = self._bridge.snapshot()['error']144 if transport_error:145 self._state, self._error = 'failed', transport_error146 self._exit_reason = 'transport_error'147 elif not requested and natural_exit:148 self._state = 'failed'149 self._error = 'agent process exited with code ' + str(natural_exit)150 self._exit_reason = 'process_crash'151 elif requested:152 self._state, self._exit_reason = 'stopped', 'stop_requested'153 else:154 self._state, self._exit_reason = 'completed', 'program_returned'155156 def _notify_exit(self) -> None:157 if not self._exit_notified:158 self._on_exit(self._state, self._error)159 self._exit_notified = True160161 def _supervise(self) -> None:162 deadline = None163 try:164 while self._process.is_alive():165 if self._bridge.snapshot()['error'] is not None:166 self.request_stop()167 if self._stop.is_set():168 if deadline is None:169 deadline = time.monotonic() + self._grace170 elif time.monotonic() >= deadline:171 break172 self._process.join(SUPERVISOR_POLL_SECONDS)173 requested = self._stop_requested174 # Capture the original exit before tree cleanup changes a still-live child.175 natural_exit = self._process.exitcode176 self._reap()177 self._record_exit(requested, natural_exit)178 except Exception as exc:179 _log.warning('Handled failure in _supervise', exc_info=True)180 with self._lock:181 self._state, self._error = 'failed', 'AGENT_CLEANUP_FAILED'182 self._exit_reason = 'cleanup_failed'183 self._cleanup_error = exc184 finally:185 self._stop.set()186 try:187 self._notify_exit()188 except Exception as exc:189 _log.warning('Handled failure in _supervise', exc_info=True)190 self._cleanup_error = exc191 try:192 self._bridge.close()193 except Exception as exc:194 _log.warning('Handled failure in _supervise', exc_info=True)195 self._cleanup_error = exc196197 def close(self) -> None:198 """Reap the owned process family and IPC; retain ownership if cleanup must retry."""199 with self._close_lock:200 if self._closed:201 return202 self.request_stop()203 if self._supervisor.ident is not None:204 self._supervisor.join(SUPERVISOR_JOIN_BASE_SECONDS + 3 * self._grace)205 if self._supervisor.is_alive():206 raise ContractError('agent process supervisor did not close')207 else:208 # Partial start: the supervisor never took ownership.209 self._reap()210 # Reattempt cleanup after any supervisor failure, without dropping the family.211 if self._cleanup_error is not None:212 self._reap()213 self._bridge.close()214 if self._supervisor.ident is not None:215 self._notify_exit()216 if self._model_setup_write is not None:217 self._model_setup_write.close()218 self._family.close()219 if self._source_workspace is not None:220 self._source_workspace.close()221 self._exitcode = self._process.exitcode222 self._process.close()223 self._cleanup_error = None224 with self._lock:225 self._closed, self._state = True, 'closed'226227 def __init__(228 self,229 request_tool: Callable[[Dict[str, Any]], Future[Any]],230 configuration: Dict[str, Any],231 limits: TransportLimits,232 *,233 program: Optional[Callable[..., Any]] = None,234 agent_factory: Optional[AgentFactory] = None,235 launch: Optional[Mapping[str, Any]] = None,236 startup_timeout: float = DEFAULT_STARTUP_TIMEOUT_SECONDS,237 shutdown_grace: float = DEFAULT_SHUTDOWN_GRACE_SECONDS,238 request_work: Optional[Callable[[], Future[Any]]] = None,239 on_exit: Optional[ExitCallback] = None,240 work_client: Optional[RuntimeClient] = None,241 local_model: bool = False,242 ) -> None:243 configuration = copy_json(configuration)244 if type(configuration) is not dict:245 raise ContractError('process requires JSON configuration and trusted program')246 if agent_factory is not None and (not callable(agent_factory) or program is not None):247 raise ContractError('supply one trusted Agent factory, without a raw program')248 self._uses_template = program is None and 'template' in configuration249 if agent_factory is not None and not self._uses_template:250 raise ContractError('Agent factory requires Agent configuration')251 if self._uses_template:252 from sim_platform.agents.carrier.program import ConfiguredAgentProgram253254 program = ConfiguredAgentProgram(agent_factory)255 elif program is None:256 program = idle_program257 self._uses_template = self._uses_template and agent_factory is None258 self._template_grants: Set[str] = set(259 configuration.get('template', {}).get('granted_tools', ())260 )261 if not callable(program):262 raise ContractError('process requires JSON configuration and trusted program')263 for value in (startup_timeout, shutdown_grace):264 if type(value) not in (int, float) or not isfinite(value) or value <= 0:265 raise ContractError('process timeouts must be positive and finite')266 self._on_exit = on_exit or (lambda status, reason: None)267 self._lock = RLock()268 self._close_lock = RLock()269 self._exit_notified = False270 self._family_reaped = False271 self._exit_reason = None272 self._stop_requested = False273 self._closed = False274 self._source_workspace = launch.get("workspace") if launch is not None else None275 self._model_configured = False276 self._model_setup_write = None277 self._error = None278 self._state = 'preparing'279 self._grace = shutdown_grace280 self._exitcode = None281 self._pid: Optional[int] = None282 self._cleanup_error = None283 context = multiprocessing.get_context('spawn')284 self._ready, self._stop, self._running = (285 ProcessSignal(context),286 ProcessSignal(context),287 ProcessSignal(context),288 )289 self._family_ready = ProcessSignal(context)290 self._launch = ProcessSignal(context)291 self._family = create_process_family()292 model_setup_read: Optional[Connection] = None293 if local_model:294 model_setup_read, self._model_setup_write = context.Pipe(duplex=False)295 startup_read, startup_write = context.Pipe(duplex=False)296 request_read, request_write = context.Pipe(duplex=False)297 response_read, response_write = context.Pipe(duplex=False)298 self._bridge = ToolPipeBridge(299 request_tool, request_read, response_write, limits, request_work=request_work300 )301 process: Callable[..., BaseProcess] = context.Process302 process_options: Dict[str, Any] = {}303 if launch is not None:304 from .processes.interpreter import InterpreterProcess305306 process = InterpreterProcess307 process_options = dict(308 executable=launch["executable"],309 project=launch["project"],310 sandbox=launch.get("sandbox"),311 )312 self._executable = launch["executable"] if launch is not None else sys.executable313 self._process = process(314 **process_options,315 target=run_program,316 name='agent-program',317 args=(318 request_write,319 response_read,320 self._ready,321 self._stop,322 self._running,323 limits,324 program,325 configuration,326 work_client,327 model_setup_read,328 startup_write,329 self._family_ready,330 self._launch,331 ),332 )333 self._supervisor = Thread(target=self._supervise, name='agent-process-supervisor')334 started = time.monotonic()335 try:336 self._process.start()337 self._pid = self._process.pid338 self._family.attach(self._process)339 self._await_family(started + startup_timeout)340 self._family.confirm()341 self._launch.set()342 startup_write.close()343 if model_setup_read is not None:344 model_setup_read.close()345 # Do not retain peer endpoints: process death must wake bridge IO.346 request_write.close()347 response_read.close()348 self._bridge.start()349 self._supervisor.start()350 self._await_initialization(started + startup_timeout, startup_read)351 self._startup_seconds = time.monotonic() - started352 except Exception as exc:353 request_write.close()354 response_read.close()355 cleanup_after_error(exc, self.close)356 raise357 finally:358 if work_client is not None:359 work_client.close() # Release parent duplicates after success or partial start.360 if model_setup_read is not None:361 model_setup_read.close()362 startup_read.close()363 startup_write.close()364365 def configure_model(self, profile: ModelProfile, secret: str, provider: ModelProvider) -> None:366 """Dedicated setup pipe: selected credentials never enter JSON config/logs/argv."""367 with self._lock:368 if self._model_setup_write is None or self._model_configured or self._state != 'ready':369 raise ContractError('configure one Agent-local model before Play')370 if self._uses_template and not TEMPLATE_MODEL_GRANTS <= self._template_grants:371 raise ContractError(372 'built-in template requires model.request and control.finish grants'373 )374 from sim_platform.models.credentials import validate_secret375376 # A local model server may need no key; only that provider may omit one.377 keyless = secret == '' and profile.provider == 'openai-compatible'378 key = '' if keyless else validate_secret(secret)379 self._model_setup_write.send((profile, key, provider))380 self._model_configured = True381 self._model_setup_write.close()382383 def snapshot(self) -> Dict[str, Any]:384 with self._lock:385 return dict(386 state=self._state,387 executable=self._executable,388 pid=self._pid,389 exitcode=self._exitcode,390 error=self._error,391 startup_seconds=getattr(self, '_startup_seconds', None),392 transport=self._bridge.snapshot(),393 exit_reason=self._exit_reason,394 stop_requested=self._stop_requested,395 cleanup_pending=self._cleanup_error is not None,396 process_family=self._family.name,397 pause_mode='capability_gate',398 )399400 def play(self) -> Dict[str, Any]:401 with self._lock:402 if self._stop.is_set() or self._closed:403 # Confirm the unavailable gate without pretending the child404 # resumed. Existing accepted orders retain their lifecycle.405 return dict(participant='decision', phase='unavailable', pid=self._pid)406 self._state = 'running'407 self._running.set()408 return dict(participant='decision', phase='gate_open', pid=self._pid)409410 def pause(self) -> Dict[str, Any]:411 with self._lock:412 if not self._stop.is_set():413 self._running.clear()414 self._state = 'paused'415 return dict(participant='decision', phase='gate_closed', pid=self._pid)