diff --git a/services/runner/src/engines/sandbox_agent.ts b/services/runner/src/engines/sandbox_agent.ts index 3daae64111..0b4a55bc36 100644 --- a/services/runner/src/engines/sandbox_agent.ts +++ b/services/runner/src/engines/sandbox_agent.ts @@ -1,2029 +1,32 @@ /** - * sandbox-agent harness driver. + * Public sandbox-agent engine facade. * - * Drives a coding harness (Pi, Claude Code, ...) over the Agent Client Protocol (ACP) - * through the `sandbox-agent` daemon, instead of the bespoke Pi SDK calls in the pi - * engine. It serves the same /run contract (AgentRunRequest -> AgentRunResult), so the - * Python side stays thin and the choice of harness/sandbox is config, not new code. - * - * Per invoke (cold), mirroring the shipped code-evaluator DaytonaRunner pattern: - * - * SandboxAgent.start({ sandbox: local({ env }) | daytona({ create }) }) - * -> createSession({ agent: , cwd, model }) - * -> write AGENTS.md into cwd - * -> session.prompt([{ type: "text", text }]) - * -> accumulate ACP `agent_message_chunk` text + build the trace - * -> destroySandbox() - * - * Two orthogonal axes swap independently: the sandbox (where the daemon runs) and the - * harness (which engine). The ACP boundary is daemon-to-harness; the service-to-sandbox-agent - * hop stays harness-agnostic behind the Harness port. - * - * Session keep-alive (flag-gated, off by default) splits the per-invoke work into - * `acquireEnvironment` (session-scoped: sandbox, mount, session, MCP wiring) and `runTurn` - * (per-turn: otel run, prompt, usage, trace). `runSandboxAgent` composes them exactly as - * before (acquire -> runTurn -> destroy), so with the flag off behavior is byte-identical. - * The dispatch in `server.ts` reuses the two halves to continue a live session across a turn - * boundary. See docs/design/agent-workflows/projects/session-keepalive/plan.md. - * - * Tracing is built here from the ACP event stream (see tracing/otel.ts createSandboxAgentOtel), - * so it is uniform across every harness and always nests under the caller's /invoke - * span. stdout is reserved for the JSON result (see cli.ts); logs go to stderr. + * Runtime policy and lifecycle implementation live in the sibling + * `sandbox_agent/` modules. Keeping this entrypoint small makes the engine's + * supported surface obvious to the CLI, server, and tests. */ -import { rmSync } from "node:fs"; - -import { apiBase } from "../apiBase.ts"; - -import { SandboxAgent, InMemorySessionPersistDriver } from "sandbox-agent"; - -import { - createSandboxAgentOtel, - TOOL_NOT_EXECUTED_PAUSED, -} from "../tracing/otel.ts"; -import { - localRelayHost, - sandboxRelayHost, - startToolRelay, - type RelayExecutionGuard, -} from "../tools/relay.ts"; -import { - ApprovalResponder, - ApprovedExecutionGrants, - ConversationDecisions, - extractApprovalDecisions, - extractClientToolOutputs, - type ClientToolOutcome, - type Responder, -} from "../responder.ts"; -import type { ClientToolRelay } from "../tools/client-tool-relay.ts"; -import { - buildClientToolRelay, - createToolCallCorrelationIndex, -} from "./sandbox_agent/client-tools.ts"; -import { - type AgentRunRequest, - type AgentRunResult, - type EmitEvent, - type HarnessCapabilities, - type ToolCallbackContext, - type ToolPermission, - resolvePromptText, - resolveRunSessionId, -} from "../protocol.ts"; -import { - assert, - assertRequiredCapabilities, - probeCapabilities, -} from "./sandbox_agent/capabilities.ts"; -import { createAcpFetch } from "./sandbox_agent/acp-fetch.ts"; -import { buildDaemonEnv, resolveDaemonBinary } from "./sandbox_agent/daemon.ts"; -import { - createCookieFetch, - prepareDaytonaPiAssets, - DAYTONA_PI_DIR, -} from "./sandbox_agent/daytona.ts"; -import { conciseError } from "./sandbox_agent/errors.ts"; -import { buildSessionMcpServers } from "./sandbox_agent/mcp.ts"; -import { applyModel } from "./sandbox_agent/model.ts"; -import { findSwallowedPiError } from "./sandbox_agent/pi-error.ts"; -import { - buildPiExtensionEnv, - prepareLocalPiAssets, - writeOtlpAuthFile, -} from "./sandbox_agent/pi-assets.ts"; -import { - uploadToolMcpAssets, - type ToolMcpAssets, -} from "./sandbox_agent/tool-mcp-assets.ts"; -import { advertisedToolSpecs } from "../tools/public-spec.ts"; -import { buildRelayExecutionGuard } from "./sandbox_agent/relay-guard.ts"; -import { - PendingApprovalLatch, - permissionsFromRequest, -} from "../permission-plan.ts"; -import { - attachPermissionResponder, - type ParkedApprovalGateType, -} from "./sandbox_agent/acp-interactions.ts"; -import { - PAUSED, - PendingApprovalPauseController, -} from "./sandbox_agent/pause.ts"; -import { - createRunLimits, - resolveRunLimits, -} from "./sandbox_agent/run-limits.ts"; -import { - createInteraction, - resolveInteraction, - buildWorkflowReferences, -} from "../sessions/interactions.ts"; -import { claimSessionOwnership, REPLICA_ID } from "../sessions/alive.ts"; -import { - teardownDisposition, - type TeardownReason, -} from "./sandbox_agent/teardown.ts"; -import { buildSandboxProvider } from "./sandbox_agent/provider.ts"; -import { DaytonaReconnectTerminalError } from "./sandbox_agent/daytona-provider.ts"; -import { - buildRunPlan, - type BuildRunPlanDeps, - type RunPlan, -} from "./sandbox_agent/run-plan.ts"; -import { priorMessages } from "./sandbox_agent/transcript.ts"; -import { resolveRunUsage } from "./sandbox_agent/usage.ts"; -import { prepareWorkspace } from "./sandbox_agent/workspace.ts"; -import { - signSessionMountCredentials, - mountStorage, - mountStorageRemote, - unmountStorage, - discoverTunnelEndpoint, - mountHarnessSessionDirs, - harnessSessionMounts, - storeReachableFromSandbox, - type MountCredentials, -} from "./sandbox_agent/mount.ts"; -import { - hydrateHarnessSessionFromDurable, - syncHarnessSessionDurable, -} from "./sandbox_agent/session-continuity-durable.ts"; -import { - readStoredSandboxPointer, - clearSandboxPointer, - writeSandboxPointer, -} from "./sandbox_agent/sandbox-reconnect.ts"; -import { - assertLocalRunnerOwnership, - eligibleAgentSessionId, - nextTurnIndex, - sessionContinuityStore, - type SessionContinuityStore, -} from "./sandbox_agent/session-continuity.ts"; -import { resolvesToLocalProvider } from "./sandbox_agent/session-pool.ts"; +export { + acquireEnvironment, + destroyInFlightSandboxes, + resolveKeepaliveMount, +} from "./sandbox_agent/environment.ts"; +export { + sendLastMessageOnly, + type AcquireEnvironmentResult, + type ParkedApproval, + type ResumeApprovalInput, + type RunTurnOptions, + type SandboxAgentDeps, + type SessionEnvironment, +} from "./sandbox_agent/runtime-contracts.ts"; +export { + runSandboxAgent, + shouldPark, +} from "./sandbox_agent/engine.ts"; +export { runTurn } from "./sandbox_agent/run-turn.ts"; export { buildTurnText, messageTranscript, } from "./sandbox_agent/transcript.ts"; export { toAcpMcpServers } from "./sandbox_agent/mcp.ts"; - -function log(message: string): void { - process.stderr.write(`[sandbox-agent] ${message}\n`); -} - -/** Extract the run credential from the OTLP export headers (initial value, constant for the run). */ -function runCredential(request: AgentRunRequest): string { - const headers = (request.telemetry?.exporters?.otlp?.headers ?? {}) as Record< - string, - string - >; - return (headers["authorization"] ?? headers["Authorization"] ?? "").trim(); -} - -function serverPermissionsFromRequest( - request: AgentRunRequest, -): ReadonlyMap { - const permissions = new Map(); - for (const server of request.mcpServers ?? []) { - if (server.permission !== undefined) { - permissions.set(server.name, server.permission); - } - } - return permissions; -} - -type Log = (message: string) => void; -const LOCAL_DURABLE_CWD_ENOTCONN_REMOUNT_LIMIT = 1; - -// In-flight sandbox handles, by run. A process KILL (docker stop / SIGTERM / OOM mid-run) skips -// the per-run teardown — so a shutdown signal handler (see `server.ts`) drains this set to -// best-effort delete any still-running sandbox before exit. Remote (Daytona) sandboxes that even a -// signal can never reach (SIGKILL/OOM) self-reap via the lifecycle reapers in `provider.ts`. -const inFlightSandboxes = new Set<{ - destroy: (opts?: { reason?: TeardownReason }) => Promise; -}>(); - -/** - * Best-effort delete every sandbox currently mid-run, bounded so it can never hang shutdown. - * Called from the process signal handler so `docker stop` reaps remote sandboxes instead of - * leaking them. Each delete is independent and its own failure is swallowed; the whole sweep is - * raced against `timeoutMs` so a slow Daytona API call cannot block the exit. - */ -export async function destroyInFlightSandboxes( - timeoutMs = 5000, - reason: TeardownReason = "shutdown-in-flight", -): Promise { - const pending = [...inFlightSandboxes]; - if (pending.length === 0) return; - const sweep = Promise.allSettled( - pending.map((environment) => - Promise.resolve(environment.destroy({ reason })).catch(() => {}), - ), - ); - await Promise.race([ - sweep, - new Promise((resolve) => setTimeout(resolve, timeoutMs)), - ]); -} - -function shouldSuppressPausedToolCallUpdate( - update: unknown, - pause: PendingApprovalPauseController, -): boolean { - const frame = update as - | { sessionUpdate?: unknown; toolCallId?: unknown } - | undefined; - const kind = frame?.sessionUpdate; - if (kind !== "tool_call" && kind !== "tool_call_update") return false; - const toolCallId = - typeof frame?.toolCallId === "string" ? frame.toolCallId : undefined; - return pause.isPausedToolCall(toolCallId); -} - -const CLAUDE_STRICT_DEPLOYMENTS = new Set([ - "custom", - "bedrock", - "vertex", - "vertex_ai", -]); - -function applyClaudeConnectionEnv( - env: Record, - request: AgentRunRequest, - acpAgent: string, - logger: Log, -): void { - if (acpAgent !== "claude") return; - - // Disable the Claude Agent SDK's Tool-Search feature for every Claude run. The bundled - // SDK defaults Tool-Search ON, which makes Claude DEFER the `agenta-tools` MCP tools and - // call them before their `inputSchema` is loaded — so it emits an empty `input: {}` and - // tools-with-args (reference workflows, commit_revision) never receive their arguments. - // Our tool count is small, so deferral buys nothing and only strips the schema. The SDK - // treats only `false`/`0`/`no`/`off` as off, so the string must be "false" (not "0"/"100"). - // This is applied after `buildDaemonEnv`'s clear and is not in `KNOWN_PROVIDER_ENV_VARS`, - // so it is never stripped, and it reaches the Daytona sandbox like `ANTHROPIC_BASE_URL`. - env.ENABLE_TOOL_SEARCH = "false"; - - const deployment = request.deployment; - const selectedModel = request.model; - const baseUrl = request.endpoint?.baseUrl; - if (baseUrl) { - env.ANTHROPIC_BASE_URL = baseUrl; - logger(`claude base_url: ${baseUrl}`); - } - - if (deployment === "bedrock") { - env.CLAUDE_CODE_USE_BEDROCK = "1"; - const region = request.endpoint?.region; - if (region) { - env.AWS_REGION = region; - env.AWS_DEFAULT_REGION ??= region; - } - } else if (deployment === "vertex" || deployment === "vertex_ai") { - env.CLAUDE_CODE_USE_VERTEX = "1"; - } - - if ( - selectedModel && - (baseUrl || (deployment && CLAUDE_STRICT_DEPLOYMENTS.has(deployment))) - ) { - env.ANTHROPIC_MODEL = selectedModel; - env.ANTHROPIC_CUSTOM_MODEL_OPTION = selectedModel; - logger( - `claude model=${selectedModel} deployment=${deployment ?? ""}`, - ); - } -} - -/** - * Whether a requested-but-unsettable model fails the run (F-007). Strict by default on every - * harness path: a user who picks a model either runs that model or sees a loud error, never a - * silent (often pricier) fallback to the harness default. `AGENTA_AGENT_MODEL_STRICT=false` is - * the explicit opt-out that restores the legacy warn-and-fallback behavior. A run that requests - * no model is unaffected either way — it keeps the harness default. - */ -function modelResolutionStrict(): boolean { - return process.env.AGENTA_AGENT_MODEL_STRICT !== "false"; -} - -export interface SandboxAgentDeps extends BuildRunPlanDeps { - startSandboxAgent?: typeof SandboxAgent.start; - createPersist?: () => InMemorySessionPersistDriver; - createOtel?: typeof createSandboxAgentOtel; - buildDaemonEnv?: typeof buildDaemonEnv; - resolveDaemonBinary?: typeof resolveDaemonBinary; - buildSandboxProvider?: typeof buildSandboxProvider; - createCookieFetch?: typeof createCookieFetch; - createAcpFetch?: typeof createAcpFetch; - prepareWorkspace?: typeof prepareWorkspace; - prepareDaytonaPiAssets?: typeof prepareDaytonaPiAssets; - uploadToolMcpAssets?: typeof uploadToolMcpAssets; - probeCapabilities?: typeof probeCapabilities; - applyModel?: typeof applyModel; - startToolRelay?: typeof startToolRelay; - localRelayHost?: typeof localRelayHost; - sandboxRelayHost?: typeof sandboxRelayHost; - signSessionMountCredentials?: typeof signSessionMountCredentials; - mountStorage?: typeof mountStorage; - mountStorageRemote?: typeof mountStorageRemote; - unmountStorage?: typeof unmountStorage; - discoverTunnelEndpoint?: typeof discoverTunnelEndpoint; - /** Per-harness transcript mounts (remote only; see mount.ts). */ - mountHarnessSessionDirs?: typeof mountHarnessSessionDirs; - responderFactory?: (request: AgentRunRequest) => Responder; - resolveRunLimits?: typeof resolveRunLimits; - createRunLimits?: typeof createRunLimits; - /** Session-continuity store override (tests inject their own; default is the process singleton). */ - sessionContinuityStore?: SessionContinuityStore; - /** Durable read-back/write-forward of the continuity store (tests inject fakes). */ - hydrateHarnessSessionFromDurable?: typeof hydrateHarnessSessionFromDurable; - syncHarnessSessionDurable?: typeof syncHarnessSessionDurable; - /** Durable read/write of the sandbox pointer, for the remote reconnect ladder. */ - readStoredSandboxPointer?: typeof readStoredSandboxPointer; - clearSandboxPointer?: typeof clearSandboxPointer; - writeSandboxPointer?: typeof writeSandboxPointer; - /** - * Resolve `{replicaId, ownerReplicaId}` for a session-owned local-sandbox run, so - * `acquireEnvironment` can fail loudly instead of silently cold-starting on a non-owner - * replica. The default claims the `owner` affinity key via the coordination plane and reads - * back the actual owner (`claimSessionOwnership`); tests inject their own. `authorization` is - * the run credential (the claim authenticates as the invoke caller). - */ - resolveLocalRunnerOwner?: ( - sessionId: string, - authorization: string, - ) => Promise<{ replicaId: string; ownerReplicaId: string | undefined }>; - log?: Log; -} - -async function defaultResolveLocalRunnerOwner( - sessionId: string, - authorization: string, -): Promise<{ replicaId: string; ownerReplicaId: string | undefined }> { - // No credential ⇒ the claim would 401; treat as "no known owner" (pass), never worse than today. - if (!authorization) { - return { replicaId: REPLICA_ID, ownerReplicaId: undefined }; - } - return claimSessionOwnership(sessionId, authorization); -} - -function isTransportEndpointDisconnected(err: unknown): boolean { - const message = String(err instanceof Error ? err.message : err); - const code = - typeof err === "object" && err !== null && "code" in err - ? String((err as { code?: unknown }).code) - : ""; - return ( - code === "ENOTCONN" || - message.includes("ENOTCONN") || - message.includes("Transport endpoint is not connected") - ); -} - -function containsTransportEndpointDisconnected(value: unknown): boolean { - const seen = new Set(); - - const visit = (current: unknown): boolean => { - if (typeof current === "string") { - return isTransportEndpointDisconnected(current); - } - if (current instanceof Error) { - return isTransportEndpointDisconnected(current); - } - if (!current || typeof current !== "object") { - return false; - } - if (seen.has(current)) { - return false; - } - seen.add(current); - - const code = - "code" in current ? String((current as { code?: unknown }).code) : ""; - if (code === "ENOTCONN") { - return true; - } - - if (Array.isArray(current)) { - return current.some(visit); - } - return Object.values(current as Record).some(visit); - }; - - return visit(value); -} - -/** - * Race sentinel: a run-limits deadline (total/idle/TTFB/per-tool-call) tripped mid-turn. Distinct - * from `PAUSED` so the prompt race can tell a human pause (keep the session) from a wedge deadline - * (end the turn as an error, letting the caller's teardown reclaim the sandbox). - */ -const RUN_LIMIT_TRIPPED = Symbol("run-limit-tripped"); - -/** - * The per-turn sink the session-lifetime listeners demux into. `runTurn` swaps a fresh one in - * at turn start (`env.currentTurn`) and the dispatch clears it at turn end. The `sandbox-agent` - * listener registries are plain Sets — an event with no listener is dropped and a permission - * request with no listener is CANCELLED — so the listeners stay attached for the session's whole - * life and route into whichever turn is active, with no detach/attach window between turns. - */ -interface CurrentTurn { - run: ReturnType; - pause: PendingApprovalPauseController; - toolRelay?: { ready?: Promise; stop: () => Promise }; - /** Route a session/update for the active turn (suppress + handleUpdate + pause re-sweep). */ - handleUpdate: (update: unknown) => void; - /** Route a permission reverse-RPC for the active turn (built by attachPermissionResponder). */ - onPermissionRequest?: (req: unknown) => void; -} - -/** - * A permission gate that paused the turn and can be answered later on the SAME live session. - * Recorded for a Claude ACP permission gate (keep-alive slice 2) or a Pi ACP permission gate - * (Pi approval parking: the gate rides the extension's `ctx.ui.confirm` onto the same ACP - * permission plane). NOT recorded for a client-tool MCP pause — that cannot be answered across - * a turn boundary and stays on the cold path. Existence of this record is what makes the - * dispatch park a paused session in `awaiting_approval` instead of tearing it down. - */ -export interface ParkedApproval { - /** Which gate paused; the dispatch resumes only a recognized type and treats others as cold. */ - gateType: ParkedApprovalGateType; - /** The ACP permission-request id, answered later via `session.respondPermission`. */ - permissionId: string; - /** The gated tool call's id — matched against the incoming approval envelope's toolCallId. */ - toolCallId: string; - /** The gated tool name (logging + the durable interaction row); never its args, in logs. */ - toolName: string | undefined; - /** The gated call's original args, used to seed the resume turn's trace/egress tool span. */ - args: unknown; - /** The durable interaction row token, resolved on the answer via the onResolveInteraction hook. */ - interactionToken: string; - /** The held original `prompt()` promise; the resume awaits it after `respondPermission`. */ - promptPromise?: Promise; -} - -/** Answer a parked Claude ACP permission gate on the live session (the keep-alive resume input). */ -export interface ResumeApprovalInput { - permissionId: string; - reply: "once" | "reject"; - toolCallId: string; - toolName: string | undefined; - args: unknown; - interactionToken: string; - promptPromise?: Promise; -} - -/** Per-turn options for `runTurn`. Absent (flag off / cold) means today's byte-identical path. */ -export interface RunTurnOptions { - /** A live continuation: send only the new user text instead of the full cold transcript. */ - continuation?: boolean; - /** - * The session was rehydrated via `session/load` (the patched `resumeSession`), so the harness - * already holds the prior turns natively. Like `continuation`, the prompt is only the new user - * text; `buildTurnText` must not run. Distinct field from `continuation` because the two arrive - * through different acquire paths (live pool checkout vs a fresh cold acquire that loaded an - * old session) — `runTurn` treats them identically for the text-selection decision. - */ - loaded?: boolean; - /** - * Keep-alive approval park mode: on a Claude ACP permission gate the pause keeps the session - * alive (no settle/abort/destroy) so a later resume can answer it. A non-parkable pause (Pi - * relay, client tool) still tears down exactly as today, so this is safe to set on any eligible - * keep-alive turn. - */ - approvalParkMode?: boolean; - /** A live approval resume: answer the parked gate and stream the continued prompt's events. */ - resume?: ResumeApprovalInput; -} - -/** - * Send only the new user text (not the full cold transcript) when the harness already holds the - * prior turns: a live continuation, or a session rehydrated via `session/load`. `runTurn` calls - * this, so a test that pins it pins the shipped decision. - */ -export function sendLastMessageOnly(opts: RunTurnOptions): boolean { - return Boolean(opts.continuation || opts.loaded); -} - -/** - * A session-scoped environment that can serve many turns. Everything expensive to build lives - * here (sandbox, session, internal tool-MCP server, mounted cwd, relay/temp dirs); `destroy()` - * is the one complete idempotent teardown the pool, the shutdown handler, and the cold path all - * call. Per-turn state rides `currentTurn`, swapped in by `runTurn`. - */ -export interface SessionEnvironment { - plan: RunPlan; - logger: Log; - deps: SandboxAgentDeps; - sandbox: any; - session: any; - sessionId: string; - model: string | undefined; - capabilities: HarnessCapabilities; - strictModel: boolean; - toolCallIndex: ReturnType; - /** The current turn's client-tool relay, read by the deferred ref baked into the MCP server. */ - clientToolRelayRef: { current?: ClientToolRelay }; - mcpAbort: AbortController; - runAgentDir: string | undefined; - otlpAuthFilePath: string | undefined; - mountCreds: MountCredentials | null; - /** The mount's owning project id (keep-alive pool key FALLBACK scope, preferred is - * `runContext.project.id`); undefined when there is no mount. */ - mountProjectId?: string; - /** This acquire resumed the harness's native session via `session/load` (not cold). */ - loadedFromContinuity: boolean; - /** A remote, session-owned run whose sandbox can be parked (warm) rather than deleted at end. */ - resumable: boolean; - /** The conversation turn index this acquire's continuity record was read/written at. */ - continuityTurnIndex: number | undefined; - // Mutable teardown/turn state shared across acquire, runTurn, and destroy. - sessionDestroyRequested: boolean; - mountedCwd: string | undefined; - durableCwdSafeToDelete: boolean; - workspace: { cleanup: () => Promise } | undefined; - runtimeRemount: Promise | undefined; - closeToolMcp: (() => Promise) | undefined; - currentTurn?: CurrentTurn; - /** - * The unique ACP tool-call ids the LAST completed turn emitted (reset at each turn start). - * The keep-alive dispatch folds them into the expected next-history fingerprint at park time, - * so a tool-using turn still matches its own continuation (the FE keeps assistant tool parts). - */ - lastTurnToolCallIds: string[]; - /** - * The Claude ACP permission gate the LAST turn paused on, or undefined. Set only for a harness - * ACP permission gate, reset at each turn start; the dispatch reads it after a paused turn to - * decide whether to park in `awaiting_approval` and, on the next request, how to resume. - */ - parkedApproval?: ParkedApproval; - /** - * How many Claude ACP permission gates resolved to pendingApproval THIS turn (reset at turn - * start). More than one means parallel gates the single-gate resume cannot answer, so the - * dispatch does not park (tears down cold as today). - */ - approvalGateCount: number; - destroyed: boolean; - /** Complete, idempotent teardown selected from the typed teardown reason. */ - destroy: (opts?: { reason?: TeardownReason }) => Promise; - /** End the active turn: clear the current-turn sink (called before a park). */ - clearTurn: () => void; -} - -export type AcquireEnvironmentResult = - | { ok: true; env: SessionEnvironment } - | { ok: false; error: string }; - -/** - * Sign the session's durable mount up front so keep-alive can build a pool key (the mount's - * owning `projectId`, the FALLBACK project scope when the run carries no service-stamped - * `runContext.project.id`) and credential epoch without acquiring the whole environment. Returns - * exactly what the sign yielded: `null` when there is no session/credential to sign with, or - * the sign returned no usable mount (store unconfigured, 503, ephemeral fallback). The caller - * threads the result — null included — into `acquireEnvironment` as `presignedMount`, so the - * mount is signed exactly once per run on every path. A null result no longer forces a cold run - * on its own: the request still parks when the run context supplied a project scope, and only - * skips parking when NEITHER source yields one (`poolKeyFor` returns null). - */ -export async function resolveKeepaliveMount( - request: AgentRunRequest, - deps: SandboxAgentDeps = {}, -): Promise { - const logger = deps.log ?? log; - const sessionForMount = request.sessionId?.trim(); - const runCred = runCredential(request); - if (!sessionForMount || !runCred) return null; - const signMount = - deps.signSessionMountCredentials ?? signSessionMountCredentials; - return signMount(sessionForMount, { - apiBase: apiBase(), - authorization: runCred, - log: logger, - }); -} - -/** - * Build the session-scoped environment: sign the mount, build the run plan, start the sandbox, - * mount the durable cwd, prepare the workspace, probe capabilities, wire the internal tool-MCP - * server, and open the ACP session. Session-lifetime `onEvent`/`onPermissionRequest` listeners - * are attached once here and demux into `env.currentTurn`. - * - * Finalizers register incrementally on `env` as each resource is acquired; a mid-acquire failure - * runs `env.destroy()` (which null-checks every resource, so a half-built environment cannot - * leak) and returns `{ ok: false }`, mirroring today's shared teardown. When `presignedMount` is - * supplied (the keep-alive cold path already signed to build the pool key) the initial sign is - * skipped so the mount is signed once per run. - */ -export async function acquireEnvironment( - request: AgentRunRequest, - deps: SandboxAgentDeps = {}, - signal?: AbortSignal, - presignedMount?: MountCredentials | null, -): Promise { - const logger = deps.log ?? log; - const acquireStartedAt = Date.now(); - const timingLog = (stage: string, startedAt: number, fields = ""): void => { - const sandboxId = environment?.sandbox?.sandboxId ?? "-"; - const sessionId = - environment?.sessionId ?? request.sessionId?.trim() ?? "-"; - logger( - `[timing] stage=${stage} ms=${Math.round(Date.now() - startedAt)} sandbox=${sandboxId} session=${sessionId}${fields}`, - ); - }; - - // Local multi-runner fails loudly. Session-owned + local-sandbox only (a non-session run - // has no cross-replica identity to protect, and a remote sandbox has no runner-local pooled - // state to protect it FROM). The resolver claims the `owner` affinity key and reads the actual - // owner back; a KNOWN different owner throws (never a silent wrong-host cold start). - const continuitySessionForOwnership = request.sessionId?.trim(); - if ( - continuitySessionForOwnership && - resolvesToLocalProvider(request.sandbox) - ) { - const { replicaId, ownerReplicaId } = await ( - deps.resolveLocalRunnerOwner ?? defaultResolveLocalRunnerOwner - )(continuitySessionForOwnership, runCredential(request)); - try { - assertLocalRunnerOwnership( - continuitySessionForOwnership, - replicaId, - ownerReplicaId, - ); - } catch (err) { - return { ok: false, error: conciseError(err, request.harness ?? "") }; - } - } - - // Sign BEFORE buildRunPlan so the prefix is available for the durable cwd derivation. - // Inputs (sessionId, apiBase, credential) are independent of the plan. Best-effort: null on - // failure leaves durableCwd undefined and buildRunPlan falls back to the ephemeral path. - const sessionForMount = request.sessionId?.trim(); - const runCred = runCredential(request); - const signMount = - deps.signSessionMountCredentials ?? signSessionMountCredentials; - let mountCreds: MountCredentials | null = - presignedMount !== undefined - ? presignedMount - : sessionForMount && runCred - ? await signMount(sessionForMount, { - apiBase: apiBase(), - authorization: runCred, - log: logger, - }) - : null; - - // Derive the durable cwd from the sign prefix (one source of truth, both providers). - // local: /tmp/agenta/ — daytona: /home/sandbox/agenta/ - // is already "mounts//", so no extra slug is needed. - let durableCwd: string | undefined; - if (mountCreds?.prefix) { - const isDaytonaReq = - (request.sandbox ?? process.env.SANDBOX_AGENT_PROVIDER ?? "local") === - "daytona"; - durableCwd = isDaytonaReq - ? `/home/sandbox/agenta/${mountCreds.prefix}` - : `/tmp/agenta/${mountCreds.prefix}`; - } - - const planResult = buildRunPlan(request, { - sandboxProvider: deps.sandboxProvider, - createLocalCwd: deps.createLocalCwd, - createDaytonaCwd: deps.createDaytonaCwd, - durableCwd, - resolveSkillDirs: deps.resolveSkillDirs, - log: logger, - }); - if (!planResult.ok) return { ok: false, error: planResult.error }; - const plan = planResult.plan; - - // Clear-then-apply (Security rule 5): on a managed run (credentialMode "env") the daemon - // inherits NONE of the sidecar's own provider keys, so only the resolved `plan.secrets` are - // present and an inherited key for another provider cannot leak. For runtime_provided/none/ - // un-migrated runs the harness uses its own login, so the inherited keys stay. - const clearProviderEnv = plan.credentialMode === "env"; - const env = (deps.buildDaemonEnv ?? buildDaemonEnv)(plan.acpAgent, { - clearProviderEnv, - }); - Object.assign(env, plan.secrets); // apply only the resolved provider keys - applyClaudeConnectionEnv(env, request, plan.acpAgent, logger); - const strictModel = modelResolutionStrict(); - // Pi self-instruments locally: propagate the trace context + public tool metadata into Pi - // via the Agenta extension. Tool execution always relays back to this runner, which keeps - // private specs, scoped env, callback endpoints, and callback auth in memory. - // local Pi's OTLP bearer rides a runner-written 0600 file, never a plain env var — - // Daytona never receives telemetry env here at all (`!plan.isDaytona` gates it off above). - const otlpAuthFilePath = - plan.isPi && !plan.isDaytona ? `${plan.relayDir}.otlp-auth` : undefined; - const otlpAuthorization = - request.telemetry?.exporters?.otlp?.headers?.authorization; - if (otlpAuthFilePath && otlpAuthorization) { - writeOtlpAuthFile(otlpAuthFilePath, otlpAuthorization, logger); - } - const piExtEnv = plan.isPi - ? buildPiExtensionEnv(request, !plan.isDaytona, { - relayDir: plan.relayDir, - usageOutPath: plan.usageOutPath, - otlpAuthFilePath, - builtinGatingActive: plan.builtinGatingActive, - builtinGrants: plan.builtinGrants, - // The materialized skill names (author + forced `_agenta.*`) so Pi's own agent span - // records which skills loaded; local Pi self-instruments, so the runner's sandbox-agent - // otel has no span to stamp here. - skills: plan.skillDirs.map((s) => s.name), - }) - : {}; - Object.assign(env, piExtEnv); // local daemon inherits it; daytona gets it via envVars - logger( - `tools=${plan.toolSpecs.length} executableTools=${plan.executableToolSpecs.length} ` + - `piPublicTools=${piExtEnv.AGENTA_AGENT_TOOLS_PUBLIC_SPECS ? "yes" : "no"}`, - ); - if (!plan.isPi && plan.isDaytona) { - const omittedClientTools = plan.toolSpecs - .filter((spec) => spec.kind === "client") - .map((spec) => spec.name); - if (omittedClientTools.length > 0) { - logger( - `omitting client tools from Daytona stdio MCP shim: ${omittedClientTools.join(", ")}`, - ); - } - } - // undefined is fine: the local provider runs its own resolution and errors clearly. - const binaryPath = (deps.resolveDaemonBinary ?? resolveDaemonBinary)(); - const runAgentDir = prepareLocalPiAssets({ plan, env, log: logger }); - - logger(`harness=${plan.harness} sandbox=${plan.sandboxId} cwd=${plan.cwd}`); - - // The resolved model ref as it reaches the runner (key NAMES only, never values) — the one - // line that answers "what model/provider/deployment/credential did this run actually use". - logger( - `resolved model=${request.model ?? ""} provider=${request.provider ?? ""} ` + - `deployment=${request.deployment ?? ""} ` + - `connection=${request.connection ? `${request.connection.mode}:${request.connection.slug ?? "-"}` : ""} ` + - `secretKeys=[${Object.keys(request.secrets ?? {}).join(",")}]`, - ); - - // The shared client-tool relay reference (the deferred ref baked into the MCP server reads it; - // each turn's `runTurn` sets `.current`). A `tools/call` can only arrive during a prompt — - // long after the relay is wired — so the server captures this reference and it resolves to the - // real relay before any call lands. - const clientToolRelayRef: { current?: ClientToolRelay } = {}; - const deferredClientToolRelay: ClientToolRelay = { - onClientTool: (req) => - clientToolRelayRef.current - ? clientToolRelayRef.current.onClientTool(req) - : Promise.resolve("deny" as ClientToolOutcome), - onPause: (req) => clientToolRelayRef.current?.onPause?.(req), - }; - - // Aborts any in-flight loopback `tools/call` (a paused Claude client tool) on pause/teardown, - // so its handler is torn down deterministically and cannot write a result after the turn ends. - const mcpAbort = new AbortController(); - - const environment: SessionEnvironment = { - plan, - logger, - deps, - sandbox: undefined, - session: undefined, - sessionId: resolveRunSessionId(request, ""), - model: undefined, - capabilities: {}, - strictModel, - toolCallIndex: createToolCallCorrelationIndex(), - clientToolRelayRef, - mcpAbort, - runAgentDir, - otlpAuthFilePath, - mountCreds, - mountProjectId: mountCreds?.projectId, - loadedFromContinuity: false, - resumable: false, - continuityTurnIndex: undefined, - sessionDestroyRequested: false, - mountedCwd: undefined, - durableCwdSafeToDelete: true, - // Local runs get a plain rmSync cleanup for the throwaway cwd; Daytona has none on this host. - workspace: plan.isDaytona - ? undefined - : { - cleanup: async () => - rmSync(plan.cwd, { recursive: true, force: true }), - }, - runtimeRemount: undefined, - closeToolMcp: undefined, - currentTurn: undefined, - lastTurnToolCallIds: [], - parkedApproval: undefined, - approvalGateCount: 0, - destroyed: false, - destroy: async () => {}, - clearTurn: () => {}, - }; - - environment.clearTurn = () => { - environment.currentTurn = undefined; - }; - - // The one complete, idempotent teardown — the same steps the old per-run `finally` ran, in the - // same order. Every resource is null-checked, so it is safe after a partial acquire and safe to - // call twice (the guard returns on a second call). It must never throw. - environment.destroy = async (opts?: { reason?: TeardownReason }) => { - if (environment.destroyed) return; - environment.destroyed = true; - await environment.runtimeRemount?.catch(() => {}); - inFlightSandboxes.delete(environment); - await environment.currentTurn?.toolRelay?.stop().catch(() => {}); - // Teardown backstop: destroy any in-flight loopback `tools/call` before closing the server. - environment.mcpAbort.abort(); - await environment.closeToolMcp?.().catch(() => {}); - // Graceful `session/cancel` BEFORE tearing down the daemon, or the ACP adapter subprocess - // reparents to PID 1 and never exits. Skip if the pause path already sent it. - if (environment.session && !environment.sessionDestroyRequested) - await environment.sandbox - ?.destroySession?.(environment.session.id) - .catch(() => {}); - const disposition = teardownDisposition(opts?.reason ?? "failed-turn"); - let parked = false; - if ( - disposition === "stop" && - plan.isDaytona && - environment.sandbox?.pauseSandbox - ) { - const sandboxLogId = environment.sandbox.sandboxId ?? plan.sandboxId; - try { - await environment.sandbox.pauseSandbox(); - parked = true; - logger(`parked sandbox=${sandboxLogId}`); - } catch (err) { - logger( - `pause failed sandbox=${sandboxLogId}: ${conciseError(err, plan.harness)}`, - ); - } - } - if (!parked) await environment.sandbox?.destroySandbox().catch(() => {}); - await environment.sandbox?.dispose().catch(() => {}); - // Unmount the durable cwd BEFORE removing the dir: data lives in the store, only the host - // mountpoint is torn down. If unmount is not CONFIRMED gone, skip the delete: rmSync must - // never run against a possibly-live FUSE mount into the durable store. - if (environment.mountedCwd) { - environment.durableCwdSafeToDelete = await ( - environment.deps.unmountStorage ?? unmountStorage - )(environment.mountedCwd, { log }).catch(() => false); - } - if (!environment.durableCwdSafeToDelete) { - logger( - `durable cwd unmount not confirmed, skipping workspace cleanup cwd=${plan.cwd}`, - ); - } else { - await environment.workspace?.cleanup().catch(() => {}); - } - // The per-run Agenta agent dir (skills isolation) is throwaway; remove it too. - if (environment.runAgentDir) - rmSync(environment.runAgentDir, { recursive: true, force: true }); - // Backstop: the extension deletes this on read; remove it here too in case the harness never - // started (or crashed before reading it), so the bearer never lingers. - if (environment.otlpAuthFilePath) - rmSync(environment.otlpAuthFilePath, { force: true }); - // Remove the per-run skills temp root the materializer created (success or error). - plan.skillsCleanup(); - }; - - // --- local durable cwd mount helpers (session-scoped, close over environment) ------ // - const mountLocalDurableCwd = async (reason: string): Promise => { - if (!environment.mountCreds || plan.isDaytona) return false; - logger( - `local durable cwd mount (${reason}) session=${sessionForMount} cwd=${plan.cwd}`, - ); - if ( - await (deps.mountStorage ?? mountStorage)( - plan.cwd, - environment.mountCreds, - { - log: logger, - }, - ) - ) { - environment.mountedCwd = plan.cwd; - return true; - } - return false; - }; - let localDurableCwdEnotconnRemounts = 0; - const reSignAndRemountLocalCwd = async (): Promise => { - if (!sessionForMount || !runCred || plan.isDaytona) return false; - if ( - localDurableCwdEnotconnRemounts >= - LOCAL_DURABLE_CWD_ENOTCONN_REMOUNT_LIMIT - ) { - logger( - `local durable cwd ENOTCONN remount limit reached session=${sessionForMount} cwd=${plan.cwd}`, - ); - return false; - } - localDurableCwdEnotconnRemounts += 1; - logger( - `local durable cwd ENOTCONN session=${sessionForMount} cwd=${plan.cwd}; re-signing and remounting`, - ); - const fresh = await signMount(sessionForMount, { - apiBase: apiBase(), - authorization: runCred, - log: logger, - }); - if (!fresh) { - logger( - `local durable cwd re-sign returned no credentials session=${sessionForMount}`, - ); - return false; - } - environment.mountCreds = fresh; - return mountLocalDurableCwd("enotconn-retry"); - }; - const remountLocalCwdAfterRuntimeEnotconn = (event: unknown): void => { - if (plan.isDaytona || !environment.mountCreds || !environment.mountedCwd) - return; - if ( - environment.runtimeRemount || - !containsTransportEndpointDisconnected(event) - ) - return; - logger( - `local durable cwd ENOTCONN observed in ACP event session=${sessionForMount} cwd=${plan.cwd}; re-signing and remounting`, - ); - environment.runtimeRemount = reSignAndRemountLocalCwd().catch((err) => { - logger( - `local durable cwd runtime remount failed session=${sessionForMount}: ${conciseError(err, plan.harness)}`, - ); - return false; - }); - }; - - try { - // Persist events in-process so a follow-up turn can resume by session id. - const persist = - deps.createPersist?.() ?? new InMemorySessionPersistDriver(); - const startSandboxAgent = - deps.startSandboxAgent ?? - ((options: Parameters[0]) => - SandboxAgent.start(options)); - const sandboxProvider = (deps.buildSandboxProvider ?? buildSandboxProvider)( - plan.sandboxId, - env, - binaryPath, - piExtEnv, - plan.secrets, - plan.sandboxPermission, - ); - const startOptions = { - sandbox: sandboxProvider, - persist, - // Propagate caller cancellation (a client disconnect on the streaming HTTP edge) so an - // in-flight run aborts instead of finishing unobserved. `destroy` still disposes. - ...(signal ? { signal } : {}), - // Long-timeout undici dispatcher so a paused HITL turn is not reaped by undici's default - // headersTimeout; Daytona additionally carries the per-sandbox auth cookie. - fetch: plan.isDaytona - ? (deps.createCookieFetch ?? createCookieFetch)() - : (deps.createAcpFetch ?? createAcpFetch)(), - }; - // A stored sandbox id is trusted: reconnect it by id and let reconnect converge its network - // policy to this run's plan. Any reconnect failure falls through to a fresh create. Snapshot - // and image drift are accepted as per-conversation version pinning, not grounds for a rebuild. - const storedSandboxPointer = - plan.isDaytona && sessionForMount && runCred - ? await (deps.readStoredSandboxPointer ?? readStoredSandboxPointer)( - sessionForMount, - { authorization: runCred, log: logger }, - ) - : undefined; - if (storedSandboxPointer) { - const sandboxStartStartedAt = Date.now(); - try { - environment.sandbox = await startSandboxAgent({ - ...startOptions, - sandboxId: storedSandboxPointer.sandboxId, - }); - logger( - `reconnected sandbox=${storedSandboxPointer.sandboxId} session=${sessionForMount}`, - ); - } catch (err) { - logger( - `reconnect failed sandbox=${storedSandboxPointer.sandboxId}, creating fresh: ${conciseError(err, plan.harness)}`, - ); - if ( - err instanceof DaytonaReconnectTerminalError && - sessionForMount && - runCred - ) { - // The post-hydrate write later in acquire is authoritative. This clear only prevents - // repeated doomed reconnects if acquire fails before reaching that write. Hydrate - // first: after a runner restart the in-memory store is behind the durable - // latest_turn_index, and an unhydrated guard token would be rejected as stale. - await ( - deps.hydrateHarnessSessionFromDurable ?? - hydrateHarnessSessionFromDurable - )( - sessionForMount, - plan.harness, - deps.sessionContinuityStore ?? sessionContinuityStore, - { authorization: runCred, log: logger }, - ); - await (deps.clearSandboxPointer ?? clearSandboxPointer)( - sessionForMount, - nextTurnIndex( - sessionForMount, - deps.sessionContinuityStore ?? sessionContinuityStore, - ), - { authorization: runCred, log: logger }, - ); - } - } finally { - timingLog("sandbox_start", sandboxStartStartedAt, " mode=reconnect"); - } - } - if (!environment.sandbox) { - const sandboxStartStartedAt = Date.now(); - try { - environment.sandbox = await startSandboxAgent(startOptions); - } finally { - timingLog("sandbox_start", sandboxStartStartedAt, " mode=create"); - } - } - environment.resumable = Boolean(plan.isDaytona && sessionForMount); - // Track the live handle so a shutdown signal handler can delete it if `destroy` is skipped by - // a process KILL; removed in `destroy` on every normal exit so it is never double-deleted. - if (environment.sandbox) inFlightSandboxes.add(environment); - - // On Daytona, push the harness login, the extension, and AGENTS.md into the remote sandbox. - // For a non-Pi harness with executable tools, also push the in-sandbox stdio MCP shim - // assets (bundle + public-specs file): a non-Pi harness in the sandbox cannot reach the - // runner-loopback HTTP MCP channel, so the harness's ACP adapter spawns the uploaded shim - // as the internal stdio MCP server instead. Uploaded unconditionally for non-Pi (the - // capability probe runs later; a harness that turns out to lack MCP fails loud in - // `assertRequiredCapabilities` below). Pi delivers via its extension; local non-Pi uses - // the loopback HTTP channel — neither needs this. The upload helper THROWS when the shim - // cannot be delivered (fail loud — this path requires it). - let internalToolMcp: ToolMcpAssets | undefined; - if (plan.isDaytona) { - await (deps.prepareDaytonaPiAssets ?? prepareDaytonaPiAssets)({ - sandbox: environment.sandbox, - plan, - log: logger, - }); - if (!plan.isPi && plan.executableToolSpecs.length > 0) { - internalToolMcp = await ( - deps.uploadToolMcpAssets ?? uploadToolMcpAssets - )( - environment.sandbox, - plan.toolMcpDir, - advertisedToolSpecs(plan.executableToolSpecs), - logger, - ); - } - } - - // Durable cwd: mount BEFORE createSession (so the session opens inside it) and BEFORE - // workspace materialization (so AGENTS.md, harness files, and skills land in the durable - // prefix instead of being hidden under the FUSE mount). - if (environment.mountCreds && !plan.isDaytona) { - await mountLocalDurableCwd("initial"); - } - if (environment.mountCreds && plan.isDaytona) { - const mountsStartedAt = Date.now(); - try { - // Mount against the store's own endpoint when the sandbox can reach it (public S3); fall - // back to the tunnel only for an in-network store. No tunnel + in-network store => skip. - const storeEndpoint = environment.mountCreds.endpoint; - const endpoint = storeReachableFromSandbox(storeEndpoint) - ? undefined - : ((await (deps.discoverTunnelEndpoint ?? discoverTunnelEndpoint)({ - log: logger, - })) ?? undefined); - const canMount = storeReachableFromSandbox(storeEndpoint) || !!endpoint; - if ( - canMount && - (await (deps.mountStorageRemote ?? mountStorageRemote)( - environment.sandbox, - plan.cwd, - environment.mountCreds, - { - endpoint, - log: logger, - }, - )) - ) { - logger(`remote durable cwd active for session=${sessionForMount}`); - } - // Per-harness session/transcript-dir mounts, remote-only by construction (this whole - // branch is `plan.isDaytona`) — local runs never reach here, so they stay mount-free/ - // byte-identical. Opt-out via env, default on wherever a durable cwd mount is active (no - // separate credential/session-id path from the cwd mount). - if ( - canMount && - sessionForMount && - runCred && - process.env.AGENTA_SESSION_HARNESS_MOUNTS !== "false" - ) { - const dirs = harnessSessionMounts( - plan.acpAgent, - "/home/sandbox", - DAYTONA_PI_DIR, - ); - await (deps.mountHarnessSessionDirs ?? mountHarnessSessionDirs)( - environment.sandbox, - sessionForMount, - dirs, - endpoint, - { - apiBase: apiBase(), - authorization: runCred, - log: logger, - }, - ); - } - } finally { - timingLog("mounts", mountsStartedAt); - } - } - - const prepareWorkspaceStartedAt = Date.now(); - try { - environment.workspace = await (deps.prepareWorkspace ?? prepareWorkspace)( - { - sandbox: environment.sandbox, - plan, - log: logger, - }, - ); - } catch (err) { - if ( - !plan.isDaytona && - environment.mountCreds && - isTransportEndpointDisconnected(err) && - (await reSignAndRemountLocalCwd()) - ) { - logger( - `retrying workspace preparation after local durable cwd remount`, - ); - environment.workspace = await ( - deps.prepareWorkspace ?? prepareWorkspace - )({ - sandbox: environment.sandbox, - plan, - log: logger, - }); - } else { - throw err; - } - } finally { - timingLog("prepare_workspace", prepareWorkspaceStartedAt); - } - - // Sandbox-start invariant: `startSandboxAgent` must hand back a usable handle. - assert( - environment.sandbox && - typeof environment.sandbox.createSession === "function", - `sandbox provider '${plan.sandboxId}' returned no usable sandbox handle`, - ); - - // Probe what this harness supports and branch on capabilities, not on the harness name. - const probeCapabilitiesStartedAt = Date.now(); - let probed; - try { - probed = await (deps.probeCapabilities ?? probeCapabilities)( - environment.sandbox, - plan.acpAgent, - ); - } finally { - timingLog("probe_capabilities", probeCapabilitiesStartedAt); - } - const capabilities = probed.capabilities; - environment.capabilities = capabilities; - - // Fail loud (A7): a run that REQUIRES a capability the harness lacks errors specifically - // rather than silently dropping the behavior. - assertRequiredCapabilities({ - harness: plan.harness, - isPi: plan.isPi, - probed, - toolSpecs: plan.toolSpecs, - log: logger, - }); - - const sessionMcp = await buildSessionMcpServers({ - isPi: plan.isPi, - capabilities, - harness: plan.harness, - isDaytona: plan.isDaytona, - toolSpecs: plan.toolSpecs, - userMcpServers: request.mcpServers, - relayDir: plan.relayDir, - clientToolRelay: deferredClientToolRelay, - signal: mcpAbort.signal, - // The uploaded in-sandbox stdio MCP shim assets, set only on Daytona + non-Pi + - // executable-tools; advertises the gateway tools the loopback channel cannot reach - // from inside the sandbox. No server to close for this entry (the harness owns the - // shim process), so `sessionMcp.close` semantics are unchanged. - internalToolMcp, - log: logger, - }); - // Close the internal gateway-tool MCP server (if one started) when the session is destroyed. - environment.closeToolMcp = sessionMcp.close; - - // If this harness authored the conversation's most recent turn (staleness-guarded) and we - // still remember its native `agentSessionId`, seed the fresh persist driver with a synthetic - // record and resume-by-id so the patched `resumeSession` reaches `session/load` instead of - // `session/new`. Any failure inside `resumeSession` already degrades to a plain new session - // internally (the patch's own `catch {}` around `loadRemoteSession`), so this call is safe to - // attempt unconditionally whenever we have an eligible id — worst case it is exactly today's - // cold `createSession`. - const continuitySessionKey = request.sessionId?.trim(); - const continuityStore = - deps.sessionContinuityStore ?? sessionContinuityStore; - // Seed the in-memory store from the durable row before consulting it, so a resume after a - // runner restart (in-memory map lost) still sees the prior turn's eligibility. No-op (and - // cheap) when the store already has a live in-process record. - if (continuitySessionKey && runCred) { - await ( - deps.hydrateHarnessSessionFromDurable ?? - hydrateHarnessSessionFromDurable - )(continuitySessionKey, plan.harness, continuityStore, { - authorization: runCred, - log: logger, - }); - } - const priorAgentSessionId = continuitySessionKey - ? eligibleAgentSessionId( - continuitySessionKey, - plan.harness, - continuityStore, - ) - : undefined; - const localSessionId = continuitySessionKey - ? `${continuitySessionKey}:${plan.harness}` - : undefined; - // The index THIS turn will occupy once it completes: recorded post-turn against the SAME - // index read here, so a turn that authors turn N leaves the store agreeing with itself. - environment.continuityTurnIndex = continuitySessionKey - ? nextTurnIndex(continuitySessionKey, continuityStore) - : undefined; - // Daytona only: a local run must not overwrite a conversation's remote pointer (switching - // sandboxes mid-conversation would strand the parked Daytona instance). - if (plan.isDaytona && sessionForMount && runCred) { - const liveSandboxId = environment.sandbox?.sandboxId ?? plan.sandboxId; - const pointerWriteOutcome = await ( - deps.writeSandboxPointer ?? writeSandboxPointer - )( - sessionForMount, - { - sandboxId: liveSandboxId, - turnIndex: environment.continuityTurnIndex ?? 0, - }, - { authorization: runCred, log: logger }, - ); - logger( - `sandbox pointer write ${pointerWriteOutcome} session=${sessionForMount} sandbox=${liveSandboxId}`, - ); - } - let loadedFromContinuity = false; - if (priorAgentSessionId && localSessionId) { - await persist.updateSession({ - id: localSessionId, - agent: plan.acpAgent, - agentSessionId: priorAgentSessionId, - lastConnectionId: "", - createdAt: Date.now(), - sessionInit: { cwd: plan.cwd, mcpServers: sessionMcp.servers }, - }); - const createSessionStartedAt = Date.now(); - try { - environment.session = - await environment.sandbox.resumeSession(localSessionId); - loadedFromContinuity = - environment.session.agentSessionId === priorAgentSessionId; - logger( - `[continuity] session/load attempted session=${continuitySessionKey} ` + - `harness=${plan.harness} loaded=${loadedFromContinuity}`, - ); - } catch (err) { - logger( - `[continuity] resumeSession failed, falling back to cold createSession: ` + - `${conciseError(err, plan.harness)}`, - ); - } finally { - timingLog("create_session", createSessionStartedAt, " mode=load"); - } - } - environment.loadedFromContinuity = loadedFromContinuity; - if (!environment.session) { - const createSessionStartedAt = Date.now(); - try { - environment.session = await environment.sandbox.createSession({ - ...(localSessionId ? { id: localSessionId } : {}), - agent: plan.acpAgent, - cwd: plan.cwd, - sessionInit: { cwd: plan.cwd, mcpServers: sessionMcp.servers }, - }); - } finally { - timingLog("create_session", createSessionStartedAt, " mode=create"); - } - } - environment.sessionId = resolveRunSessionId( - request, - environment.session.id, - ); - - // Resolve the model first: when the harness rejects the requested id and keeps its own - // default, `model` is undefined and the chat span is labelled "chat". - environment.model = await (deps.applyModel ?? applyModel)( - environment.session, - request.model, - logger, - { strict: strictModel }, - ); - - // Session-lifetime listeners: attach ONCE, each demuxing into the active turn's sink. They - // outlive any single turn, so the routing lives in dedicated non-throwing helpers below. - environment.session.onEvent((event: any) => - routeSessionEventToActiveTurn( - environment, - remountLocalCwdAfterRuntimeEnotconn, - event, - ), - ); - environment.session.onPermissionRequest((req: any) => - routePermissionRequestToActiveTurn(environment, req), - ); - - timingLog("acquire_total", acquireStartedAt); - return { ok: true, env: environment }; - } catch (err) { - const error = conciseError(err, plan.harness, request.provider); - // Mirror today's shared teardown: no otel exists yet during acquire, so there is no partial - // trace to flush — just run the incrementally-registered finalizers and surface the error. - await environment.destroy({ reason: "failed-turn" }); - return { ok: false, error }; - } -} - -/** - * Route one harness event into the active turn's sink. - * - * Data flow: the ACP session emits an event -> we demux it -> the active turn - * (`environment.currentTurn`) consumes the update. The session listener is attached ONCE and - * outlives every turn, so this must never throw: the sandbox-agent registries are plain Sets and a - * thrown handler would corrupt the event stream, so any error is swallowed and logged. - * - * Steps: let the ENOTCONN watcher observe the raw event, extract the update payload (dropping events - * that carry none), record live tool_call ids for client-tool correlation, then hand the update to - * the active turn — or, between turns when no turn owns it, log and drop it. - */ -function routeSessionEventToActiveTurn( - environment: SessionEnvironment, - remountLocalCwdAfterRuntimeEnotconn: (event: unknown) => void, - event: any, -): void { - const { logger, plan } = environment; - try { - remountLocalCwdAfterRuntimeEnotconn(event); - const payload = event?.payload; - const update = payload?.params?.update ?? payload?.update; - if (!update) return; - // Record live ACP tool_call ids so a paused client_tool can correlate to Claude's bubble - // (session-scoped; a lookup CONSUMES its matched id). - environment.toolCallIndex.record(update); - const turn = environment.currentTurn; - if (turn) { - turn.handleUpdate(update); - } else { - // Between turns (parked/idle): no turn owns this event. Log and drop by decision. - logger(`[keepalive] between-turns event dropped`); - } - } catch (err) { - logger(`session onEvent handler error: ${conciseError(err, plan.harness)}`); - } -} - -/** - * Route one permission gate into the active turn's approval handler. - * - * Data flow: the harness raises a permission request -> the active turn (`environment.currentTurn`) - * decides it. Like the event listener this is attached ONCE and must never throw (a thrown handler - * would corrupt the sandbox-agent registries), so errors are swallowed and logged. - * - * Between turns no turn owns the gate. An approval park is always recorded DURING the active turn - * (the gate fires while a prompt runs, routing through currentTurn), and a parked-on-approval - * session leaves its harness suspended on that gate, so nothing new fires while parked. A gate that - * reaches here is therefore a genuine stray (e.g. a late teardown artifact): reject it by policy so - * it cannot hang. - */ -function routePermissionRequestToActiveTurn( - environment: SessionEnvironment, - req: any, -): void { - const { logger, plan } = environment; - try { - const turn = environment.currentTurn; - if (turn?.onPermissionRequest) { - turn.onPermissionRequest(req); - return; - } - logger( - `[keepalive] between-turns permission request, cancelling by policy id=${req?.id}`, - ); - void Promise.resolve( - environment.session?.respondPermission?.(req?.id, "reject"), - ).catch(() => {}); - } catch (err) { - logger( - `session onPermissionRequest handler error: ${conciseError(err, plan.harness)}`, - ); - } -} - -/** - * Run one turn against an acquired environment: start a fresh otel run, wire this turn's pause - * controller / latch / decisions / responder into `env.currentTurn`, restart the tool relay, - * send the prompt, resolve usage, and finish + flush the trace. It does NOT tear down the - * environment (the caller owns `env.destroy`). On a continuation the prompt is only the new user - * text (`buildTurnText` does not run); on a cold turn it is `plan.turnText`, exactly as before. - */ -export async function runTurn( - env: SessionEnvironment, - request: AgentRunRequest, - emit?: EmitEvent, - signal?: AbortSignal, - opts: RunTurnOptions = {}, -): Promise { - const { plan, logger, deps } = env; - const sessionId = env.sessionId; - // Reset the per-turn tool-call id record (the park folds the completed turn's ids into the - // expected next-history fingerprint). - env.lastTurnToolCallIds = []; - // Reset the per-turn approval-park bookkeeping. A fresh turn starts with no parked gate; this - // turn re-records it only if it pauses on a Claude ACP permission gate. (The dispatch has - // already captured any prior park into `opts.resume` before calling us.) - env.parkedApproval = undefined; - env.approvalGateCount = 0; - // Hoisted so the catch can flush a partial trace (mirroring the pre-split `otel?` handling — - // a createOtel throw must still return `{ ok: false }`, not propagate raw) and the finally can - // stop this turn's relay on EVERY exit path (a cleared sink must never orphan it). - let otel: ReturnType | undefined; - let activeTurn: CurrentTurn | undefined; - - // Time-based run deadlines (total/idle/TTFB/per-tool-call) for THIS turn: an idle/wedged harness - // has no deadline anywhere, so a silent or hung turn would hold its sandbox forever. Tripping a - // limit resolves the prompt race with `RUN_LIMIT_TRIPPED`, which ends the turn as an error so the - // caller's teardown (`runSandboxAgent`'s `finally`, or the keep-alive dispatch's evict-on-failure) - // reclaims the sandbox exactly as any other error does. Disposed in the `finally` on every path. - // A human pause retires the deadlines (`notePaused`): a HITL wait is legitimate, not a wedge. - const runLimits = (deps.createRunLimits ?? createRunLimits)( - (deps.resolveRunLimits ?? resolveRunLimits)(logger), - { log: logger }, - ); - let runLimitTrip: (() => void) | undefined; - let runLimitReason: string | undefined; - const runLimitTripped = new Promise((resolve) => { - runLimitTrip = resolve; - }); - runLimits.onTrip((reason) => { - runLimitReason = reason; - runLimitTrip?.(); - }); - - try { - const promptText = resolvePromptText(request); - // Cold: replay the full transcript (plan.turnText). Continuation or loaded: send only new text. - const turnText = sendLastMessageOnly(opts) ? promptText : plan.turnText; - - const run = (deps.createOtel ?? createSandboxAgentOtel)({ - harness: plan.harness, - model: env.model, - skills: plan.skillDirs.map((s) => s.name), - traceparent: request.context?.propagation?.traceparent, - baggage: request.context?.propagation?.baggage, - endpoint: request.telemetry?.exporters?.otlp?.endpoint, - authorization: request.telemetry?.exporters?.otlp?.headers?.authorization, - captureContent: request.telemetry?.capture?.content?.enabled, - emitSpans: !plan.isPi || plan.isDaytona, - // Every emitted event is a progress signal for the idle/TTFB deadlines (message/thought - // deltas, tool calls and results, usage, ...) — the one seam every harness's output flows - // through. Per-tool-call timers are driven separately from `handleUpdate` below. - emit: emit && runLimits.wrapEmit(emit), - }); - otel = run; - - run.start({ - prompt: promptText, - sessionId, - messages: [ - ...priorMessages(request), - { role: "user", content: promptText }, - ], - }); - - const pause = new PendingApprovalPauseController(() => { - // The sibling settle runs UNCONDITIONALLY, park mode or not: latch-loser tool calls - // announced before the winning gate can never execute this turn, and skipping the settle - // here would leave them as orphaned open parts whenever the dispatch later refuses the park - // (multi-gate, pool full) — `env.destroy()` does not re-run it. The exclusion keeps the - // gated (paused) call itself open, so the live resume is untouched. - run.settleOpenToolCalls( - (id) => pause.isPausedToolCall(id), - TOOL_NOT_EXECUTED_PAUSED, - ); - // Park mode: a parkable permission gate (Claude ACP or Pi ACP) recorded - // `env.parkedApproval` BEFORE firing this pause (the onUserApprovalGate hook runs before - // the single-pause latch). Keep the live session — the gated tool runs on the resume — so - // skip ONLY the mcpAbort and the destroySession. The teardown is not lost: the dispatch - // either parks the session or, if it decides not to (multi-gate, pool full), calls - // `env.destroy()` which runs them. A non-parkable pause (keep-alive off, client tool) - // never records `parkedApproval`, so it still tears down here exactly as today. - if (opts.approvalParkMode && env.parkedApproval) return; - // Abort any in-flight loopback `tools/call` (a paused Claude client tool) BEFORE the - // session teardown, so its handler cannot write a result after the turn ends. - env.mcpAbort.abort(); - env.sessionDestroyRequested = true; - return env.sandbox.destroySession?.(env.session.id); - }); - // A human pause resolves this signal exactly once, the moment the turn parks for input — the one - // place every pause path converges, so the one place to retire the run-limits deadlines for good. - void pause.signal.then(() => runLimits.notePaused()); - - // Publish this turn's sink so the session-lifetime listeners route into it. handleUpdate - // reproduces the old per-event routing (suppress paused frames, handleUpdate, pause re-sweep). - const turn: CurrentTurn = { - run, - pause, - toolRelay: undefined, - handleUpdate: (update) => { - // Per-tool-call deadline: starts on the announcement, ends on a terminal status. Tracked - // regardless of the pause-suppression below (a call already timed out must not linger just - // because a later sibling frame gets suppressed). - const rawFrame = update as { - sessionUpdate?: unknown; - toolCallId?: unknown; - status?: unknown; - }; - if (rawFrame?.sessionUpdate === "tool_call" && rawFrame.toolCallId) { - runLimits.noteToolCallStart(String(rawFrame.toolCallId)); - } else if ( - rawFrame?.sessionUpdate === "tool_call_update" && - rawFrame.toolCallId && - (rawFrame.status === "completed" || rawFrame.status === "failed") - ) { - runLimits.noteToolCallEnd(String(rawFrame.toolCallId)); - } - if (!shouldSuppressPausedToolCallUpdate(update, pause)) { - // Record the emitted tool-call ids (unique, first-seen order): the park folds them - // into the expected next-history fingerprint so a tool-using turn continues live. - const frame = update as { - sessionUpdate?: unknown; - toolCallId?: unknown; - }; - if ( - frame?.sessionUpdate === "tool_call" && - typeof frame.toolCallId === "string" && - frame.toolCallId && - !env.lastTurnToolCallIds.includes(frame.toolCallId) - ) { - env.lastTurnToolCallIds.push(frame.toolCallId); - } - run.handleUpdate(update); - // A sibling announced AFTER the pause won the latch can never execute; settle it - // immediately so the client never holds an orphaned part (idempotent re-sweep). - if (pause.active) { - run.settleOpenToolCalls( - (id) => pause.isPausedToolCall(id), - TOOL_NOT_EXECUTED_PAUSED, - ); - } - } - }, - onPermissionRequest: undefined, - }; - activeTurn = turn; - env.currentTurn = turn; - - const permissionPlan = permissionsFromRequest(request); - const storedDecisionMap = extractApprovalDecisions(request); - if (storedDecisionMap.size > 0) { - logger( - `[HITL] resume state: decisions=${JSON.stringify([...storedDecisionMap.keys()])}`, - ); - } - const decisions = new ConversationDecisions( - storedDecisionMap, - extractClientToolOutputs(request), - ); - const executionGrants = new ApprovedExecutionGrants(); - const latch = new PendingApprovalLatch(); - const responder = - deps.responderFactory?.(request) ?? - new ApprovalResponder(permissionPlan, decisions, logger); - // Every pause seeds the durable interactions plane, whichever gate paused. - const recordPendingInteraction = ( - token: string, - toolName: string | undefined, - toolArgs: unknown, - kind: "user_approval" | "client_tool" = "user_approval", - ): void => { - const cred = runCredential(request); - if (!cred) return; - const references = buildWorkflowReferences(request.runContext?.workflow); - if (!references?.workflow_revision) return; - void createInteraction( - sessionId, - request.turnId ?? "", - token, - kind, - { request: { tool: toolName ?? token, args: toolArgs }, references }, - () => cred, - ); - }; - // Transition the durable interaction row to resolved once its gate is answered. Used both by - // the cold decision-map path (via attachPermissionResponder) and the live approval resume, - // which answers the parked gate directly. It mirrors the cold path's ordering against - // `cancelStaleInteractions` (server.ts): that sweep cancels only PENDING gates of OTHER turns, - // and by resume time the human already marked this gate responded, so it is spared here too. - const resolveInteractionToken = (token: string): void => { - const cred = runCredential(request); - if (!cred) return; - if ( - !buildWorkflowReferences(request.runContext?.workflow) - ?.workflow_revision - ) - return; - void resolveInteraction(sessionId, token, () => cred); - }; - const serverPermissions = serverPermissionsFromRequest(request); - // Build the per-turn permission handler WITHOUT attaching to the live session: the - // session-lifetime `onPermissionRequest` (in acquireEnvironment) routes into it via - // `currentTurn`. A capturing shim reuses attachPermissionResponder unchanged; its - // respondPermission delegates to the real session. - attachPermissionResponder({ - session: { - onPermissionRequest: (handler: (req: unknown) => void) => { - turn.onPermissionRequest = handler; - }, - respondPermission: (id: string, reply: string) => - env.session.respondPermission(id, reply), - }, - run, - responder, - latch, - serverPermissions, - log: logger, - onPause: () => pause.pause(), - onPausedToolCall: (id) => pause.markPausedToolCall(id), - onCreateInteraction: recordPendingInteraction, - onResolveInteraction: resolveInteractionToken, - // Pi runs only: presence of the specs map turns Pi gate envelope detection on AND is how - // the runner recovers specPermission/readOnlyHint (the envelope carries identity, never - // policy). Absent for Claude, so a title collision there keeps the base path. - piToolSpecsByName: plan.isPi - ? new Map( - plan.toolSpecs.map((spec) => [ - spec.name, - { - permission: spec.permission, - readOnly: spec.readOnly, - // callRef tools only: bound paths are runner-filled at execution, so the - // approval card and decision keys must not carry the model's values for them. - contextBindings: spec.callRef - ? spec.contextBindings - : undefined, - }, - ]), - ) - : undefined, - // A resolved custom-tool allow becomes an execution grant the relay guard consumes, so - // only a dialog-approved (or policy-allowed) call ever executes from the relay dir. - onPiGateAllowed: (info) => - executionGrants.grant(info.toolName, info.args), - // Record the parkable permission gate (only in keep-alive park mode) so the dispatch can - // resume it live. Fires per pending gate (before the latch) so a parallel gate is counted; - // the single-gate resume records only the FIRST gate's answer target. `info.gateType` names - // the plane (Claude ACP vs Pi ACP) so the resume answers on the right one. - onUserApprovalGate: opts.approvalParkMode - ? (info) => { - env.approvalGateCount += 1; - if ( - env.approvalGateCount === 1 && - info.permissionId && - info.toolCallId - ) { - env.parkedApproval = { - gateType: info.gateType, - permissionId: info.permissionId, - toolCallId: info.toolCallId, - toolName: info.toolName, - args: info.args, - interactionToken: info.interactionToken, - }; - } - } - : undefined, - }); - - // Resolve the ONE client-tool seam both delivery paths share. The correlation index is wired - // for Claude only — Pi's relay toolCallId is already exact. - env.clientToolRelayRef.current = buildClientToolRelay({ - responder, - run, - latch, - pause, - recordPendingInteraction, - toolCallIndex: plan.isPi ? undefined : env.toolCallIndex, - log: logger, - }); - - // EVERY harness gets the guard: the relay dir is sandbox-writable, so a forged - // `.req.json` proves nothing about any dialog having run, and this runner-side - // re-check is the only enforcement of the hard deny boundary against forged files. - // `allow` passes and `deny` refuses identically everywhere; `ask` splits by harness — - // Pi consumes a dialog-recorded execution grant (fail-closed parity with the in-sandbox - // confirm), while a non-Pi MCP harness (Claude) passes `ask` because its own harness - // enforces the ask dialog (the rendered `mcp__agenta-tools__` ask rules + the ACP - // permission flow) before a call reaches the shim. See buildRelayExecutionGuard for the - // stated residual (a forged file can still trigger an ask-tool without a dialog there). - const relayGuard: RelayExecutionGuard = buildRelayExecutionGuard({ - isPi: plan.isPi, - permissionPlan, - executionGrants, - }); - - if (plan.useToolRelay) { - turn.toolRelay = (deps.startToolRelay ?? startToolRelay)( - plan.isDaytona - ? (deps.sandboxRelayHost ?? sandboxRelayHost)(env.sandbox, { - log: logger, - }) - : (deps.localRelayHost ?? localRelayHost)(), - plan.relayDir, - plan.toolSpecs, - request.toolCallback as ToolCallbackContext | undefined, - request.runContext, - env.clientToolRelayRef.current, - relayGuard, - { log: logger }, - ); - // Ordering invariant: the relay's stale-file sweep must complete before the - // resume's respondPermission or the fresh prompt below can cause a legitimate - // request, so nothing legitimate can predate the sweep and be swallowed as - // stale. Optional-chained so a fake relay without `ready` is tolerated, and a - // sweep failure never kills the turn. - await turn.toolRelay?.ready?.catch?.(() => {}); - } - - // The prompt promise this turn races against the pause signal. A normal/continuation turn - // sends a fresh prompt; a live approval resume answers the parked gate on the SAME session and - // continues the ORIGINAL, still-pending prompt promise (the tool then runs with its original - // byte-exact args). Either way, on a HITL pause the prompt resolves cancelled or never - // resolves, and the pause signal ends the turn. - let promptPromise: Promise; - if (opts.resume) { - // The new (resume) turn owns streaming + tracing; the environment is already wired to route - // continued events into this turn's sink (env.currentTurn was set above). Seed this run's - // trace with the parked tool call so the completing `tool_call_update` closes it and the FE - // approval part flips to output-available even if the adapter re-announces nothing. Then - // answer the gate on the live session — the original prompt continues from here. - run.handleUpdate({ - sessionUpdate: "tool_call", - toolCallId: opts.resume.toolCallId, - title: opts.resume.toolName, - kind: opts.resume.toolName, - rawInput: opts.resume.args, - }); - promptPromise = Promise.resolve(opts.resume.promptPromise); - promptPromise.catch(() => {}); - // A parked Pi dialog gate resumes on a FRESH turn whose relay and grant ledger are new; - // grant the approved call here so the extension's execute record (written right after the - // confirm resolves) passes the relay guard. Claude resumes grant too — harmlessly, no - // guard consults it. - if (opts.resume.reply === "once") { - executionGrants.grant(opts.resume.toolName, opts.resume.args); - } - await env.session.respondPermission( - opts.resume.permissionId, - opts.resume.reply, - ); - // The gate is answered: resolve the durable interaction row (the parked pending row the cold - // path would otherwise resolve via its decision map). The fresh per-turn pause controller - // starts with an EMPTY pausedToolCallIds set, so the resumed call's `tool_call_update` frames - // are no longer suppressed and stream through — the "clear pausedToolCallIds on resume" step. - resolveInteractionToken(opts.resume.interactionToken); - logger( - `[keepalive] resume answered gate reply=${opts.resume.reply} tool=${opts.resume.toolName ?? "?"}`, - ); - } else { - promptPromise = Promise.resolve( - env.session.prompt([{ type: "text", text: turnText }]), - ); - promptPromise.catch(() => {}); - } - const raced = await Promise.race([ - promptPromise, - pause.signal.then(() => PAUSED), - runLimitTripped.then(() => RUN_LIMIT_TRIPPED), - ]); - // A tripped run-limit ends the turn as an error: throw into the shared catch below so the - // trace is flushed and the caller's teardown reclaims the (wedged) sandbox. - if (raced === RUN_LIMIT_TRIPPED) { - throw new Error(runLimitReason ?? "run limit tripped"); - } - const stopReason = - raced === PAUSED || pause.active ? "paused" : (raced as any)?.stopReason; - const result = raced === PAUSED ? undefined : raced; - // A parkable pause this turn: hand the still-pending prompt promise to the parked record so a - // later resume can await the same continuation. (Set after the race so `promptPromise` exists. - // The read is asserted because the onUserApprovalGate callback set the field via an async - // mutation TS's flow analysis cannot see, so it would otherwise narrow the reset to `never`.) - const parkedThisTurn = env.parkedApproval as ParkedApproval | undefined; - if (opts.approvalParkMode && pause.active && parkedThisTurn) { - parkedThisTurn.promptPromise = promptPromise; - } - await turn.toolRelay?.stop(); - logger(`prompt stopReason=${stopReason}`); - - const usage = await resolveRunUsage({ - sandbox: env.sandbox, - usageOutPath: plan.usageOutPath, - isDaytona: plan.isDaytona, - promptResult: result, - streamUsage: run.usage(), - }); - run.setUsage(usage); - - const swallowedPiError = - plan.isPi && - !plan.isDaytona && - !run.output().trim() && - !run.events().some((e) => e.type === "tool_call") - ? findSwallowedPiError( - env.runAgentDir ?? plan.sourcePiAgentDir, - plan.cwd, - ) - : undefined; - let swallowedError: string | undefined; - if (swallowedPiError) { - swallowedError = conciseError( - new Error(swallowedPiError), - plan.harness, - request.provider, - ); - run.recordError(swallowedError, request.provider); - run.emitEvent({ type: "error", message: swallowedError }); - } - - const output = run.finish(); - await run.flush(); - - if (swallowedError) { - // A failed turn may have left a partial turn in the native transcript: the prior record - // is no longer a faithful resume point. - invalidateContinuity(sessionId, plan.harness, deps); - return { ok: false, error: swallowedError }; - } - - // Capture this harness's native session id for the next turn's setup. Only on a turn that - // actually completed (not paused mid-turn — a park has not finished authoring the turn, so - // it must not be marked authoritative) and only when the harness surfaced one. - if ( - stopReason !== "paused" && - env.continuityTurnIndex !== undefined && - sessionId && - env.session?.agentSessionId - ) { - (deps.sessionContinuityStore ?? sessionContinuityStore).record( - sessionId, - plan.harness, - env.session.agentSessionId, - env.continuityTurnIndex, - ); - // Mirror the record durably so it survives a runner restart; fire-and-forget. - const syncCred = runCredential(request); - if (syncCred) { - void (deps.syncHarnessSessionDurable ?? syncHarnessSessionDurable)( - sessionId, - plan.harness, - env.session.agentSessionId, - env.continuityTurnIndex, - { authorization: syncCred, log: logger }, - ); - } - } else if (stopReason === "paused") { - // A pause stopped mid-turn, after the harness may have written a partial turn natively. - invalidateContinuity(sessionId, plan.harness, deps); - } - - return { - ok: true, - output, - messages: output ? [{ role: "assistant", content: output }] : [], - events: emit ? [] : run.events(), - usage, - stopReason, - capabilities: { - ...env.capabilities, - streamingDeltas: !!emit && env.capabilities.streamingDeltas, - }, - sessionId, - model: env.model ?? request.model, - traceId: run.traceId(), - } as AgentRunResult; - } catch (err) { - const error = conciseError(err, plan.harness, request.provider); - otel?.recordError(error, request.provider); - otel?.emitEvent({ type: "error", message: error }); - // An aborted turn may have left a partial turn in the native transcript. - invalidateContinuity(sessionId, plan.harness, deps); - // finish() must not throw uncaught — tracing must not mask the run error. - try { - otel?.finish(); - } catch {} - await otel?.flush().catch(() => {}); - return { ok: false, error }; - } finally { - // Release every run-limits timer (idempotent, never re-arms on a late event) on EVERY path. - runLimits.dispose(); - // This turn owns its relay: stop it on EVERY exit path (the happy path already stopped it - // after the prompt; stop is safe to repeat, matching the old finally). Null it afterwards so - // a later `destroy()` — possibly after the dispatch cleared the sink — cannot double-stop or - // orphan it. - await activeTurn?.toolRelay?.stop().catch(() => {}); - if (activeTurn) activeTurn.toolRelay = undefined; - } -} - -/** - * The cold, one-turn-per-environment entry (also the flag-off path). Acquire an environment, run - * one turn, then tear the environment down — exactly as the single `try/finally` did before the - * split, so behavior here is byte-identical to pre-keep-alive. - */ -/** - * Drop the harness's continuity record after a turn that did not complete. The harness may have - * written a partial turn into its native transcript, so a later `session/load` would resume a - * history the canonical request never sent. Dropping it falls back to cold replay. - */ -function invalidateContinuity( - sessionId: string | undefined, - harness: string, - deps: SandboxAgentDeps, -): void { - if (!sessionId) return; - (deps.sessionContinuityStore ?? sessionContinuityStore).invalidate( - sessionId, - harness, - ); -} - -/** - * Whether a completed turn's environment may be parked: never on abort, client disconnect, - * pause, or failure. Session-owned streams survive disconnect WITHOUT aborting the run signal - * (server policy), so the disconnect check needs the separate `clientGone` flag. A wedged - * sandbox that failed its turn must be destroyed, not reconnected on the next one. - */ -export function shouldPark( - result: AgentRunResult, - signal: AbortSignal | undefined, - clientGone: (() => boolean) | undefined, -): boolean { - if (signal?.aborted) return false; // aborted run: destroy, do not park - if (clientGone?.()) return false; // client disconnected mid-turn: destroy, do not park - if (!result.ok) return false; // failed turn: teardown as today - if (result.stopReason === "paused") return false; // a plain pause never parks - return true; -} - -export async function runSandboxAgent( - request: AgentRunRequest, - emit?: EmitEvent, - signal?: AbortSignal, - deps: SandboxAgentDeps = {}, -): Promise { - const acquired = await acquireEnvironment(request, deps, signal); - if (!acquired.ok) return { ok: false, error: acquired.error }; - const env = acquired.env; - let result: AgentRunResult | undefined; - try { - result = await runTurn(env, request, emit, signal, { - loaded: env.loadedFromContinuity, - }); - return result; - } finally { - // `result` is undefined when runTurn threw: a failed turn, so destroy. - const cleanResumable = - env.resumable && - result !== undefined && - shouldPark(result, signal, undefined); - await env.destroy({ - reason: cleanResumable - ? "clean-resumable" - : signal?.aborted - ? "aborted" - : "failed-turn", - }); - } -} diff --git a/services/runner/src/engines/sandbox_agent/acp-fetch.ts b/services/runner/src/engines/sandbox_agent/acp-fetch.ts index 4f13d7c55b..b4f79876ad 100644 --- a/services/runner/src/engines/sandbox_agent/acp-fetch.ts +++ b/services/runner/src/engines/sandbox_agent/acp-fetch.ts @@ -58,7 +58,12 @@ export function createAcpDispatcher(): Agent { * `fetch` so the `dispatcher` option is honored regardless of how the global dispatcher is set. * The `sandbox-agent` SDK accepts a custom `fetch`; we hand it this one on every path. */ -export function createAcpFetch(dispatcher: Agent = createAcpDispatcher()): typeof fetch { - return ((input: any, init?: any) => - undiciFetch(input, { ...init, dispatcher })) as unknown as typeof fetch; +export function createAcpFetch( + dispatcher: Agent = createAcpDispatcher(), +): typeof fetch { + return ((input: Parameters[0], init?: RequestInit) => + undiciFetch( + input as unknown as Parameters[0], + { ...init, dispatcher } as unknown as Parameters[1], + )) as unknown as typeof fetch; } diff --git a/services/runner/src/engines/sandbox_agent/acp-interactions.ts b/services/runner/src/engines/sandbox_agent/acp-interactions.ts index 8a948ab009..aa2d352449 100644 --- a/services/runner/src/engines/sandbox_agent/acp-interactions.ts +++ b/services/runner/src/engines/sandbox_agent/acp-interactions.ts @@ -1,3 +1,5 @@ +import type { PermissionReply } from "sandbox-agent"; + import type { AgentEvent, ToolPermission } from "../../protocol.ts"; import { decisionToReply, @@ -18,10 +20,33 @@ import { redactContextBoundArgs } from "../../tools/relay.ts"; /** The parkable gate types a paused turn can record (the Claude ACP and Pi ACP gates). */ export type ParkedApprovalGateType = - "claude-acp-permission" | "pi-acp-permission"; + | "claude-acp-permission" + | "pi-acp-permission"; /** The permission metadata the runner recovers per tool for a Pi gate (the identity-only * envelope carries no policy). Keyed by resolved tool name. */ +interface ToolCallLike extends Record { + toolCallId?: unknown; + rawInput?: unknown; + input?: unknown; + name?: unknown; + title?: unknown; + kind?: unknown; +} + +interface PermissionRequestLike extends Record { + id?: unknown; + toolCall?: ToolCallLike; + availableReplies?: unknown; + options?: unknown; +} + +function permissionRequest(value: unknown): PermissionRequestLike { + return value && typeof value === "object" + ? (value as PermissionRequestLike) + : {}; +} + export interface PiToolSpecMeta { permission?: ToolPermission; readOnly?: boolean; @@ -31,7 +56,10 @@ export interface PiToolSpecMeta { } export interface AttachPermissionResponderInput { - session: any; + session: { + onPermissionRequest(listener: (request: unknown) => void): unknown; + respondPermission(id: string, reply: PermissionReply): Promise | void; + }; run: { emitEvent: (event: AgentEvent) => void; events?: () => AgentEvent[] }; responder: Responder; latch: PendingApprovalLatch; @@ -101,8 +129,8 @@ export function attachPermissionResponder({ onPiGateAllowed, piToolSpecsByName, }: AttachPermissionResponderInput): void { - session.onPermissionRequest((req: any) => { - void handleRequest(req).catch((err) => { + session.onPermissionRequest((value: unknown) => { + void handleRequest(permissionRequest(value)).catch((err) => { log?.(`[HITL] permission handling failed: ${errorMessage(err)}`); onPause?.(); }); @@ -115,7 +143,10 @@ export function attachPermissionResponder({ // is the Pi gate's id/args normalization in `handlePiGate`, which must happen in place so // every downstream read sees the envelope's real identity — with `rawInput` set to the // gate's REDACTED args, never the model's values for context-bound paths.) - const stampResolvedName = (toolCall: any, gate: GateDescriptor): any => { + const stampResolvedName = ( + toolCall: ToolCallLike | undefined, + gate: GateDescriptor, + ): ToolCallLike | undefined => { if (!toolCall || typeof toolCall !== "object" || !gate.toolName) return toolCall; return { ...toolCall, resolvedName: gate.toolName }; @@ -130,7 +161,7 @@ export function attachPermissionResponder({ // `onPause` (session teardown resolves the RPC as cancelled, not rejected) and the next // turn's stored decision answers the re-raised gate. const pauseUserApproval = ( - req: any, + req: PermissionRequestLike, id: string, gate: GateDescriptor, gateType: ParkedApprovalGateType, @@ -167,7 +198,7 @@ export function attachPermissionResponder({ }; const pauseClientTool = ( - req: any, + req: PermissionRequestLike, id: string, gate: GateDescriptor, spec: ToolSpecLike, @@ -207,7 +238,7 @@ export function attachPermissionResponder({ try { await session.respondPermission( id, - decisionToReply(decision, availableReplies) as any, + permissionReply(decisionToReply(decision, availableReplies)), ); } catch (err) { log?.( @@ -227,7 +258,7 @@ export function attachPermissionResponder({ try { await session.respondPermission( id, - clientToolReply(verdict, availableReplies) as any, + permissionReply(clientToolReply(verdict, availableReplies)), ); } catch (err) { log?.( @@ -255,7 +286,7 @@ export function attachPermissionResponder({ try { await session.respondPermission( id, - decisionToReply("deny", availableReplies) as any, + permissionReply(decisionToReply("deny", availableReplies)), ); } catch (err) { log?.(`[HITL] reject failed id=${id}: ${errorMessage(err)}`); @@ -272,7 +303,7 @@ export function attachPermissionResponder({ * like a relay-gate card without showing model values the execution will overwrite. */ const handlePiGate = async ( - req: any, + req: PermissionRequestLike, id: string, availableReplies: string[], envelope: PiGateEnvelope, @@ -325,7 +356,7 @@ export function attachPermissionResponder({ await replyPermission(id, verdict.kind, availableReplies); }; - async function handleRequest(req: any): Promise { + async function handleRequest(req: PermissionRequestLike): Promise { const id = stringValue(req?.id) ?? ""; const availableReplies = stringArray(req?.availableReplies); @@ -470,7 +501,7 @@ function recordedToolName( } function buildGateDescriptor( - toolCall: any, + toolCall: ToolCallLike | undefined, run: { events?: () => AgentEvent[] }, serverPermissions: ReadonlyMap, ): GateDescriptor { @@ -516,7 +547,9 @@ type ToolSpecLike = { render?: unknown; }; -function resolvedSpecOf(toolCall: any): ToolSpecLike | undefined { +function resolvedSpecOf( + toolCall: ToolCallLike | undefined, +): ToolSpecLike | undefined { const spec = toolCall?.spec ?? toolCall?.toolSpec ?? @@ -552,6 +585,13 @@ function interactionEventId(id: string, toolCallId: unknown): string { return id || stringValue(toolCallId) || ""; } +function permissionReply(value: string): PermissionReply { + if (value === "once" || value === "always" || value === "reject") { + return value; + } + throw new Error(`unsupported ACP permission reply: ${value}`); +} + function clientToolReply( verdict: Exclude, availableReplies: string[], diff --git a/services/runner/src/engines/sandbox_agent/capabilities.ts b/services/runner/src/engines/sandbox_agent/capabilities.ts index 70cc446088..f386a8da0b 100644 --- a/services/runner/src/engines/sandbox_agent/capabilities.ts +++ b/services/runner/src/engines/sandbox_agent/capabilities.ts @@ -67,31 +67,36 @@ export function toolDeliveryUnsupportedMessage( */ export function mapCapabilities( harness: string, - info: any, + info: unknown, ): ProbedCapabilities { assert( typeof harness === "string" && harness.length > 0, "mapCapabilities requires a non-empty harness id", ); - const c = info?.capabilities; + const c = + info && typeof info === "object" + ? (info as { capabilities?: unknown }).capabilities + : undefined; if (c) { assert( typeof c === "object", `probed capabilities for '${harness}' is not an object (got ${typeof c})`, ); + const flags = c as Record; return { source: "probed", capabilities: { - textMessages: c.textMessages ?? true, - images: !!c.images, - fileAttachments: !!c.fileAttachments, - mcpTools: !!c.mcpTools, - toolCalls: !!c.toolCalls, - reasoning: !!c.reasoning, - planMode: !!c.planMode, - permissions: !!c.permissions, - streamingDeltas: !!c.streamingDeltas, - sessionLifecycle: !!c.sessionLifecycle, + textMessages: + typeof flags.textMessages === "boolean" ? flags.textMessages : true, + images: !!flags.images, + fileAttachments: !!flags.fileAttachments, + mcpTools: !!flags.mcpTools, + toolCalls: !!flags.toolCalls, + reasoning: !!flags.reasoning, + planMode: !!flags.planMode, + permissions: !!flags.permissions, + streamingDeltas: !!flags.streamingDeltas, + sessionLifecycle: !!flags.sessionLifecycle, usage: true, }, }; @@ -117,8 +122,12 @@ export function mapCapabilities( } /** Probe the harness's capabilities from the daemon, falling back to static policy. */ +export interface CapabilityProbe { + getAgent?(agent: string, options: { config: true }): Promise; +} + export async function probeCapabilities( - sandbox: any, + sandbox: CapabilityProbe, harness: string, ): Promise { assert( @@ -127,7 +136,7 @@ export async function probeCapabilities( ); let probed: ProbedCapabilities; try { - const info = await sandbox.getAgent(harness, { config: true }); + const info = await sandbox.getAgent!(harness, { config: true }); probed = mapCapabilities(harness, info); } catch { probed = mapCapabilities(harness, undefined); diff --git a/services/runner/src/engines/sandbox_agent/daytona.ts b/services/runner/src/engines/sandbox_agent/daytona.ts index fdcad2d01b..b91d49e44f 100644 --- a/services/runner/src/engines/sandbox_agent/daytona.ts +++ b/services/runner/src/engines/sandbox_agent/daytona.ts @@ -8,6 +8,11 @@ import { uploadSystemPromptToSandbox, } from "./pi-assets.ts"; import { shouldUploadOwnLogin, type RunPlan } from "./run-plan.ts"; +import type { + SandboxAssetPort, + SandboxFilePort, + SandboxProcessPort, +} from "./sandbox-ports.ts"; type Log = (message: string) => void; @@ -48,7 +53,7 @@ export function daytonaEnvVars( /** Install the `pi` CLI into a Daytona sandbox (the sandbox-agent image lacks it). Best-effort. */ export async function installPiInSandbox( - sandbox: any, + sandbox: SandboxProcessPort, log: Log = () => {}, ): Promise { try { @@ -64,9 +69,13 @@ export async function installPiInSandbox( cwd: DAYTONA_PI_INSTALL_DIR, timeoutMs: 180_000, }); - if (res?.exitCode !== 0) { + const result = + res && typeof res === "object" + ? (res as { exitCode?: number | null; stderr?: unknown }) + : undefined; + if (result?.exitCode !== 0) { log( - `pi install in sandbox exit=${res?.exitCode}: ${String(res?.stderr).slice(-400)}`, + `pi install in sandbox exit=${result?.exitCode}: ${String(result?.stderr).slice(-400)}`, ); } } catch (err) { @@ -80,7 +89,7 @@ export async function installPiInSandbox( * back to any provider key in the sandbox env. */ export async function uploadPiAuthToSandbox( - sandbox: any, + sandbox: SandboxFilePort, log: Log = () => {}, ): Promise { const localDir = @@ -107,7 +116,7 @@ export async function uploadPiAuthToSandbox( } export interface PrepareDaytonaPiAssetsInput { - sandbox: any; + sandbox: SandboxAssetPort; plan: Pick< RunPlan, | "isPi" @@ -170,12 +179,17 @@ export function createCookieFetch( inner: typeof fetch = createAcpFetch(), ): typeof fetch { const jar = new Map>(); // host -> (name -> "name=value") - return async (input: any, init?: any) => { - const url = new URL(typeof input === "string" ? input : input.url); + return async ( + input: Parameters[0], + init?: RequestInit, + ): Promise => { + const url = new URL( + typeof input === "string" || input instanceof URL ? input : input.url, + ); const host = url.host; const cookies = jar.get(host); const headers = new Headers( - init?.headers ?? (typeof input !== "string" ? input.headers : undefined), + init?.headers ?? (input instanceof Request ? input.headers : undefined), ); if (cookies && cookies.size > 0) { const existing = headers.get("cookie"); @@ -184,12 +198,15 @@ export function createCookieFetch( headers.set("cookie", merged.join("; ")); } const response = await inner(input, { ...init, headers }); - const setCookies = - typeof (response.headers as any).getSetCookie === "function" - ? (response.headers as any).getSetCookie() - : response.headers.get("set-cookie") - ? [response.headers.get("set-cookie")] - : []; + const cookieHeaders = response.headers as Headers & { + getSetCookie?: () => string[]; + }; + const combinedSetCookie = response.headers.get("set-cookie"); + const setCookies = cookieHeaders.getSetCookie + ? cookieHeaders.getSetCookie() + : combinedSetCookie + ? [combinedSetCookie] + : []; if (setCookies.length) { const store = jar.get(host) ?? new Map(); for (const sc of setCookies) { diff --git a/services/runner/src/engines/sandbox_agent/engine.ts b/services/runner/src/engines/sandbox_agent/engine.ts new file mode 100644 index 0000000000..f253499dde --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/engine.ts @@ -0,0 +1,59 @@ +import type { + AgentRunRequest, + AgentRunResult, + EmitEvent, +} from "../../protocol.ts"; +import { + acquireEnvironment, +} from "./environment.ts"; +import type { SandboxAgentDeps } from "./runtime-contracts.ts"; +import { runTurn } from "./run-turn.ts"; + +/** + * Whether a completed turn's environment may be parked: never on abort, client disconnect, + * pause, or failure. Session-owned streams survive disconnect WITHOUT aborting the run signal + * (server policy), so the disconnect check needs the separate `clientGone` flag. A wedged + * sandbox that failed its turn must be destroyed, not reconnected on the next one. + */ +export function shouldPark( + result: AgentRunResult, + signal: AbortSignal | undefined, + clientGone: (() => boolean) | undefined, +): boolean { + if (signal?.aborted) return false; // aborted run: destroy, do not park + if (clientGone?.()) return false; // client disconnected mid-turn: destroy, do not park + if (!result.ok) return false; // failed turn: teardown as today + if (result.stopReason === "paused") return false; // a plain pause never parks + return true; +} + +export async function runSandboxAgent( + request: AgentRunRequest, + emit?: EmitEvent, + signal?: AbortSignal, + deps: SandboxAgentDeps = {}, +): Promise { + const acquired = await acquireEnvironment(request, deps, signal); + if (!acquired.ok) return { ok: false, error: acquired.error }; + const env = acquired.env; + let result: AgentRunResult | undefined; + try { + result = await runTurn(env, request, emit, signal, { + loaded: env.loadedFromContinuity, + }); + return result; + } finally { + // `result` is undefined when runTurn threw: a failed turn, so destroy. + const cleanResumable = + env.resumable && + result !== undefined && + shouldPark(result, signal, undefined); + await env.destroy({ + reason: cleanResumable + ? "clean-resumable" + : signal?.aborted + ? "aborted" + : "failed-turn", + }); + } +} diff --git a/services/runner/src/engines/sandbox_agent/environment-setup.ts b/services/runner/src/engines/sandbox_agent/environment-setup.ts new file mode 100644 index 0000000000..3124b0dfe8 --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/environment-setup.ts @@ -0,0 +1,280 @@ +import { rmSync } from "node:fs"; + +import { apiBase } from "../../apiBase.ts"; + + +import { +resolveRunSessionId, +type AgentRunRequest +} from "../../protocol.ts"; +import { +type ClientToolOutcome +} from "../../responder.ts"; +import type { ClientToolRelay } from "../../tools/client-tool-relay.ts"; +import { +createToolCallCorrelationIndex +} from "./client-tools.ts"; +import { buildDaemonEnv,resolveDaemonBinary } from "./daemon.ts"; +import { conciseError } from "./errors.ts"; +import { +signSessionMountCredentials, +type MountCredentials +} from "./mount.ts"; +import { +buildPiExtensionEnv, +prepareLocalPiAssets, +writeOtlpAuthFile, +} from "./pi-assets.ts"; +import { +buildRunPlan +} from "./run-plan.ts"; +import type { +SandboxAgentDeps, +SessionEnvironment, +} from "./runtime-contracts.ts"; +import { +applyClaudeConnectionEnv, +defaultResolveLocalRunnerOwner, +modelResolutionStrict, +runCredential, +} from "./runtime-policy.ts"; +import { +assertLocalRunnerOwnership +} from "./session-continuity.ts"; +import { resolvesToLocalProvider } from "./session-identity.ts"; + +function defaultLog(message: string): void { + process.stderr.write(`[sandbox-agent] ${message}\n`); +} + +export async function prepareEnvironmentSetup( + request: AgentRunRequest, + deps: SandboxAgentDeps, + presignedMount?: MountCredentials | null, +) { + const logger = deps.log ?? defaultLog; + const acquireStartedAt = Date.now(); + const timingLog = (stage: string, startedAt: number, fields = ""): void => { + const sandboxId = environment?.sandbox?.sandboxId ?? "-"; + const sessionId = + environment?.sessionId ?? request.sessionId?.trim() ?? "-"; + logger( + `[timing] stage=${stage} ms=${Math.round(Date.now() - startedAt)} sandbox=${sandboxId} session=${sessionId}${fields}`, + ); + }; + + // Local multi-runner fails loudly. Session-owned + local-sandbox only (a non-session run + // has no cross-replica identity to protect, and a remote sandbox has no runner-local pooled + // state to protect it FROM). The resolver claims the `owner` affinity key and reads the actual + // owner back; a KNOWN different owner throws (never a silent wrong-host cold start). + const continuitySessionForOwnership = request.sessionId?.trim(); + if ( + continuitySessionForOwnership && + resolvesToLocalProvider(request.sandbox) + ) { + const { replicaId, ownerReplicaId } = await ( + deps.resolveLocalRunnerOwner ?? defaultResolveLocalRunnerOwner + )(continuitySessionForOwnership, runCredential(request)); + try { + assertLocalRunnerOwnership( + continuitySessionForOwnership, + replicaId, + ownerReplicaId, + ); + } catch (err) { + return { + ok: false as const, + error: conciseError(err, request.harness ?? ""), + }; + } + } + + // Sign BEFORE buildRunPlan so the prefix is available for the durable cwd derivation. + // Inputs (sessionId, apiBase, credential) are independent of the plan. Best-effort: null on + // failure leaves durableCwd undefined and buildRunPlan falls back to the ephemeral path. + const sessionForMount = request.sessionId?.trim(); + const runCred = runCredential(request); + const signMount = + deps.signSessionMountCredentials ?? signSessionMountCredentials; + let mountCreds: MountCredentials | null = + presignedMount !== undefined + ? presignedMount + : sessionForMount && runCred + ? await signMount(sessionForMount, { + apiBase: apiBase(), + authorization: runCred, + log: logger, + }) + : null; + + // Derive the durable cwd from the sign prefix (one source of truth, both providers). + // local: /tmp/agenta/ — daytona: /home/sandbox/agenta/ + // is already "mounts//", so no extra slug is needed. + let durableCwd: string | undefined; + if (mountCreds?.prefix) { + const isDaytonaReq = + (request.sandbox ?? process.env.SANDBOX_AGENT_PROVIDER ?? "local") === + "daytona"; + durableCwd = isDaytonaReq + ? `/home/sandbox/agenta/${mountCreds.prefix}` + : `/tmp/agenta/${mountCreds.prefix}`; + } + + const planResult = buildRunPlan(request, { + sandboxProvider: deps.sandboxProvider, + createLocalCwd: deps.createLocalCwd, + createDaytonaCwd: deps.createDaytonaCwd, + durableCwd, + resolveSkillDirs: deps.resolveSkillDirs, + log: logger, + }); + if (!planResult.ok) { + return { ok: false as const, error: planResult.error }; + } + const plan = planResult.plan; + + // Clear-then-apply (Security rule 5): on a managed run (credentialMode "env") the daemon + // inherits NONE of the sidecar's own provider keys, so only the resolved `plan.secrets` are + // present and an inherited key for another provider cannot leak. For runtime_provided/none/ + // un-migrated runs the harness uses its own login, so the inherited keys stay. + const clearProviderEnv = plan.credentialMode === "env"; + const env = (deps.buildDaemonEnv ?? buildDaemonEnv)(plan.acpAgent, { + clearProviderEnv, + }); + Object.assign(env, plan.secrets); // apply only the resolved provider keys + applyClaudeConnectionEnv(env, request, plan.acpAgent, logger); + const strictModel = modelResolutionStrict(); + // Pi self-instruments locally: propagate the trace context + public tool metadata into Pi + // via the Agenta extension. Tool execution always relays back to this runner, which keeps + // private specs, scoped env, callback endpoints, and callback auth in memory. + // local Pi's OTLP bearer rides a runner-written 0600 file, never a plain env var — + // Daytona never receives telemetry env here at all (`!plan.isDaytona` gates it off above). + const otlpAuthFilePath = + plan.isPi && !plan.isDaytona ? `${plan.relayDir}.otlp-auth` : undefined; + const otlpAuthorization = + request.telemetry?.exporters?.otlp?.headers?.authorization; + if (otlpAuthFilePath && otlpAuthorization) { + writeOtlpAuthFile(otlpAuthFilePath, otlpAuthorization, logger); + } + const piExtEnv = plan.isPi + ? buildPiExtensionEnv(request, !plan.isDaytona, { + relayDir: plan.relayDir, + usageOutPath: plan.usageOutPath, + otlpAuthFilePath, + builtinGatingActive: plan.builtinGatingActive, + builtinGrants: plan.builtinGrants, + // The materialized skill names (author + forced `_agenta.*`) so Pi's own agent span + // records which skills loaded; local Pi self-instruments, so the runner's sandbox-agent + // otel has no span to stamp here. + skills: plan.skillDirs.map((s) => s.name), + }) + : {}; + Object.assign(env, piExtEnv); // local daemon inherits it; daytona gets it via envVars + logger( + `tools=${plan.toolSpecs.length} executableTools=${plan.executableToolSpecs.length} ` + + `piPublicTools=${piExtEnv.AGENTA_AGENT_TOOLS_PUBLIC_SPECS ? "yes" : "no"}`, + ); + if (!plan.isPi && plan.isDaytona) { + const omittedClientTools = plan.toolSpecs + .filter((spec) => spec.kind === "client") + .map((spec) => spec.name); + if (omittedClientTools.length > 0) { + logger( + `omitting client tools from Daytona stdio MCP shim: ${omittedClientTools.join(", ")}`, + ); + } + } + // undefined is fine: the local provider runs its own resolution and errors clearly. + const binaryPath = (deps.resolveDaemonBinary ?? resolveDaemonBinary)(); + const runAgentDir = prepareLocalPiAssets({ plan, env, log: logger }); + + logger(`harness=${plan.harness} sandbox=${plan.sandboxId} cwd=${plan.cwd}`); + + // The resolved model ref as it reaches the runner (key NAMES only, never values) — the one + // line that answers "what model/provider/deployment/credential did this run actually use". + logger( + `resolved model=${request.model ?? ""} provider=${request.provider ?? ""} ` + + `deployment=${request.deployment ?? ""} ` + + `connection=${request.connection ? `${request.connection.mode}:${request.connection.slug ?? "-"}` : ""} ` + + `secretKeys=[${Object.keys(request.secrets ?? {}).join(",")}]`, + ); + + // The shared client-tool relay reference (the deferred ref baked into the MCP server reads it; + // each turn's `runTurn` sets `.current`). A `tools/call` can only arrive during a prompt — + // long after the relay is wired — so the server captures this reference and it resolves to the + // real relay before any call lands. + const clientToolRelayRef: { current?: ClientToolRelay } = {}; + const deferredClientToolRelay: ClientToolRelay = { + onClientTool: (req) => + clientToolRelayRef.current + ? clientToolRelayRef.current.onClientTool(req) + : Promise.resolve("deny" as ClientToolOutcome), + onPause: (req) => clientToolRelayRef.current?.onPause?.(req), + }; + + // Aborts any in-flight loopback `tools/call` (a paused Claude client tool) on pause/teardown, + // so its handler is torn down deterministically and cannot write a result after the turn ends. + const mcpAbort = new AbortController(); + + const environment: SessionEnvironment = { + plan, + logger, + deps, + sandbox: undefined, + session: undefined, + sessionId: resolveRunSessionId(request, ""), + model: undefined, + capabilities: {}, + strictModel, + toolCallIndex: createToolCallCorrelationIndex(), + clientToolRelayRef, + mcpAbort, + runAgentDir, + otlpAuthFilePath, + mountCreds, + mountProjectId: mountCreds?.projectId, + loadedFromContinuity: false, + resumable: false, + continuityTurnIndex: undefined, + sessionDestroyRequested: false, + mountedCwd: undefined, + durableCwdSafeToDelete: true, + // Local runs get a plain rmSync cleanup for the throwaway cwd; Daytona has none on this host. + workspace: plan.isDaytona + ? undefined + : { + cleanup: async () => + rmSync(plan.cwd, { recursive: true, force: true }), + }, + runtimeRemount: undefined, + closeToolMcp: undefined, + currentTurn: undefined, + lastTurnToolCallIds: [], + parkedApproval: undefined, + approvalGateCount: 0, + destroyed: false, + destroy: async () => {}, + clearTurn: () => {}, + }; + + environment.clearTurn = () => { + environment.currentTurn = undefined; + }; + return { + ok: true as const, + acquireStartedAt, + binaryPath, + deferredClientToolRelay, + env, + environment, + logger, + mcpAbort, + piExtEnv, + plan, + runCred, + sessionForMount, + signMount, + strictModel, + timingLog, + }; +} diff --git a/services/runner/src/engines/sandbox_agent/environment.ts b/services/runner/src/engines/sandbox_agent/environment.ts new file mode 100644 index 0000000000..275eb816da --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/environment.ts @@ -0,0 +1,781 @@ +/** + * Sandbox-agent environment acquisition and session-scoped lifecycle. + * + * Drives a coding harness (Pi, Claude Code, ...) over the Agent Client Protocol (ACP) + * through the `sandbox-agent` daemon, instead of the bespoke Pi SDK calls in the pi + * engine. It serves the same /run contract (AgentRunRequest -> AgentRunResult), so the + * Python side stays thin and the choice of harness/sandbox is config, not new code. + * + * Per invoke (cold), mirroring the shipped code-evaluator DaytonaRunner pattern: + * + * SandboxAgent.start({ sandbox: local({ env }) | daytona({ create }) }) + * -> createSession({ agent: , cwd, model }) + * -> write AGENTS.md into cwd + * -> session.prompt([{ type: "text", text }]) + * -> accumulate ACP `agent_message_chunk` text + build the trace + * -> destroySandbox() + * + * Two orthogonal axes swap independently: the sandbox (where the daemon runs) and the + * harness (which engine). The ACP boundary is daemon-to-harness; the service-to-sandbox-agent + * hop stays harness-agnostic behind the Harness port. + * + * Session keep-alive (flag-gated, off by default) splits the per-invoke work into + * `acquireEnvironment` (session-scoped: sandbox, mount, session, MCP wiring) and `runTurn` + * (per-turn: otel run, prompt, usage, trace). `runSandboxAgent` composes them exactly as + * before (acquire -> runTurn -> destroy), so with the flag off behavior is byte-identical. + * The dispatch in `server.ts` reuses the two halves to continue a live session across a turn + * boundary. See docs/design/agent-workflows/projects/session-keepalive/plan.md. + * + * Tracing is built here from the ACP event stream (see tracing/otel.ts createSandboxAgentOtel), + * so it is uniform across every harness and always nests under the caller's /invoke + * span. stdout is reserved for the JSON result (see cli.ts); logs go to stderr. + */ +import { rmSync } from "node:fs"; + +import { apiBase } from "../../apiBase.ts"; + +import { +InMemorySessionPersistDriver, +SandboxAgent, +type SessionEvent, +type SessionPermissionRequest +} from "sandbox-agent"; + +import { +resolveRunSessionId, +type AgentRunRequest +} from "../../protocol.ts"; +import { advertisedToolSpecs } from "../../tools/public-spec.ts"; +import { createAcpFetch } from "./acp-fetch.ts"; +import { +assert, +assertRequiredCapabilities, +probeCapabilities, +} from "./capabilities.ts"; +import { DaytonaReconnectTerminalError } from "./daytona-provider.ts"; +import { +createCookieFetch, +DAYTONA_PI_DIR, +prepareDaytonaPiAssets, +} from "./daytona.ts"; +import { prepareEnvironmentSetup } from "./environment-setup.ts"; +import { conciseError } from "./errors.ts"; +import { buildSessionMcpServers } from "./mcp.ts"; +import { applyModel } from "./model.ts"; +import { +discoverTunnelEndpoint, +harnessSessionMounts, +mountHarnessSessionDirs, +mountStorage, +mountStorageRemote, +signSessionMountCredentials, +storeReachableFromSandbox, +unmountStorage, +type MountCredentials, +} from "./mount.ts"; +import { buildSandboxProvider } from "./provider.ts"; +import { +clearSandboxPointer, +readStoredSandboxPointer, +writeSandboxPointer, +} from "./sandbox-reconnect.ts"; +import { +hydrateHarnessSessionFromDurable +} from "./session-continuity-durable.ts"; +import { +eligibleAgentSessionId, +nextTurnIndex, +sessionContinuityStore +} from "./session-continuity.ts"; +import { +teardownDisposition, +type TeardownReason, +} from "./teardown.ts"; +import { +uploadToolMcpAssets, +type ToolMcpAssets, +} from "./tool-mcp-assets.ts"; +import { prepareWorkspace } from "./workspace.ts"; + +export { toAcpMcpServers } from "./mcp.ts"; +export { +buildTurnText, +messageTranscript +} from "./transcript.ts"; + +function log(message: string): void { + process.stderr.write(`[sandbox-agent] ${message}\n`); +} + +const LOCAL_DURABLE_CWD_ENOTCONN_REMOUNT_LIMIT = 1; + +// In-flight sandbox handles, by run. A process KILL (docker stop / SIGTERM / OOM mid-run) skips +// the per-run teardown — so a shutdown signal handler (see `server.ts`) drains this set to +// best-effort delete any still-running sandbox before exit. Remote (Daytona) sandboxes that even a +// signal can never reach (SIGKILL/OOM) self-reap via the lifecycle reapers in `provider.ts`. +const inFlightSandboxes = new Set<{ + destroy: (opts?: { reason?: TeardownReason }) => Promise; +}>(); + +/** + * Best-effort delete every sandbox currently mid-run, bounded so it can never hang shutdown. + * Called from the process signal handler so `docker stop` reaps remote sandboxes instead of + * leaking them. Each delete is independent and its own failure is swallowed; the whole sweep is + * raced against `timeoutMs` so a slow Daytona API call cannot block the exit. + */ +export async function destroyInFlightSandboxes( + timeoutMs = 5000, + reason: TeardownReason = "shutdown-in-flight", +): Promise { + const pending = [...inFlightSandboxes]; + if (pending.length === 0) return; + const sweep = Promise.allSettled( + pending.map((environment) => + Promise.resolve(environment.destroy({ reason })).catch(() => {}), + ), + ); + await Promise.race([ + sweep, + new Promise((resolve) => setTimeout(resolve, timeoutMs)), + ]); +} + +import type { +AcquireEnvironmentResult, +SandboxAgentDeps +} from "./runtime-contracts.ts"; +import { +containsTransportEndpointDisconnected, +isTransportEndpointDisconnected, +runCredential +} from "./runtime-policy.ts"; +/** + * Sign the session's durable mount up front so keep-alive can build a pool key (the mount's + * owning `projectId`, the FALLBACK project scope when the run carries no service-stamped + * `runContext.project.id`) and credential epoch without acquiring the whole environment. Returns + * exactly what the sign yielded: `null` when there is no session/credential to sign with, or + * the sign returned no usable mount (store unconfigured, 503, ephemeral fallback). The caller + * threads the result — null included — into `acquireEnvironment` as `presignedMount`, so the + * mount is signed exactly once per run on every path. A null result no longer forces a cold run + * on its own: the request still parks when the run context supplied a project scope, and only + * skips parking when NEITHER source yields one (`poolKeyFor` returns null). + */ +export async function resolveKeepaliveMount( + request: AgentRunRequest, + deps: SandboxAgentDeps = {}, +): Promise { + const logger = deps.log ?? log; + const sessionForMount = request.sessionId?.trim(); + const runCred = runCredential(request); + if (!sessionForMount || !runCred) return null; + const signMount = + deps.signSessionMountCredentials ?? signSessionMountCredentials; + return signMount(sessionForMount, { + apiBase: apiBase(), + authorization: runCred, + log: logger, + }); +} + +/** + * Build the session-scoped environment: sign the mount, build the run plan, start the sandbox, + * mount the durable cwd, prepare the workspace, probe capabilities, wire the internal tool-MCP + * server, and open the ACP session. Session-lifetime `onEvent`/`onPermissionRequest` listeners + * are attached once here and demux into `env.currentTurn`. + * + * Finalizers register incrementally on `env` as each resource is acquired; a mid-acquire failure + * runs `env.destroy()` (which null-checks every resource, so a half-built environment cannot + * leak) and returns `{ ok: false }`, mirroring today's shared teardown. When `presignedMount` is + * supplied (the keep-alive cold path already signed to build the pool key) the initial sign is + * skipped so the mount is signed once per run. + */ +export async function acquireEnvironment( + request: AgentRunRequest, + deps: SandboxAgentDeps = {}, + signal?: AbortSignal, + presignedMount?: MountCredentials | null, +): Promise { + const setup = await prepareEnvironmentSetup(request, deps, presignedMount); + if (!setup.ok) return setup; + const { + acquireStartedAt, + binaryPath, + deferredClientToolRelay, + env, + environment, + logger, + mcpAbort, + piExtEnv, + plan, + runCred, + sessionForMount, + signMount, + strictModel, + timingLog, + } = setup; + // The one complete, idempotent teardown — the same steps the old per-run `finally` ran, in the + // same order. Every resource is null-checked, so it is safe after a partial acquire and safe to + // call twice (the guard returns on a second call). It must never throw. + environment.destroy = async (opts?: { reason?: TeardownReason }) => { + if (environment.destroyed) return; + environment.destroyed = true; + await environment.runtimeRemount?.catch(() => {}); + inFlightSandboxes.delete(environment); + await environment.currentTurn?.toolRelay?.stop().catch(() => {}); + // Teardown backstop: destroy any in-flight loopback `tools/call` before closing the server. + environment.mcpAbort.abort(); + await environment.closeToolMcp?.().catch(() => {}); + // Graceful `session/cancel` BEFORE tearing down the daemon, or the ACP adapter subprocess + // reparents to PID 1 and never exits. Skip if the pause path already sent it. + if (environment.session && !environment.sessionDestroyRequested) + await environment.sandbox + ?.destroySession?.(environment.session.id) + .catch(() => {}); + const disposition = teardownDisposition(opts?.reason ?? "failed-turn"); + let parked = false; + if ( + disposition === "stop" && + plan.isDaytona && + environment.sandbox?.pauseSandbox + ) { + const sandboxLogId = environment.sandbox.sandboxId ?? plan.sandboxId; + try { + await environment.sandbox.pauseSandbox(); + parked = true; + logger(`parked sandbox=${sandboxLogId}`); + } catch (err) { + logger( + `pause failed sandbox=${sandboxLogId}: ${conciseError(err, plan.harness)}`, + ); + } + } + if (!parked) await environment.sandbox?.destroySandbox().catch(() => {}); + await environment.sandbox?.dispose().catch(() => {}); + // Unmount the durable cwd BEFORE removing the dir: data lives in the store, only the host + // mountpoint is torn down. If unmount is not CONFIRMED gone, skip the delete: rmSync must + // never run against a possibly-live FUSE mount into the durable store. + if (environment.mountedCwd) { + environment.durableCwdSafeToDelete = await ( + environment.deps.unmountStorage ?? unmountStorage + )(environment.mountedCwd, { log }).catch(() => false); + } + if (!environment.durableCwdSafeToDelete) { + logger( + `durable cwd unmount not confirmed, skipping workspace cleanup cwd=${plan.cwd}`, + ); + } else { + await environment.workspace?.cleanup().catch(() => {}); + } + // The per-run Agenta agent dir (skills isolation) is throwaway; remove it too. + if (environment.runAgentDir) + rmSync(environment.runAgentDir, { recursive: true, force: true }); + // Backstop: the extension deletes this on read; remove it here too in case the harness never + // started (or crashed before reading it), so the bearer never lingers. + if (environment.otlpAuthFilePath) + rmSync(environment.otlpAuthFilePath, { force: true }); + // Remove the per-run skills temp root the materializer created (success or error). + plan.skillsCleanup(); + }; + + // --- local durable cwd mount helpers (session-scoped, close over environment) ------ // + const mountLocalDurableCwd = async (reason: string): Promise => { + if (!environment.mountCreds || plan.isDaytona) return false; + logger( + `local durable cwd mount (${reason}) session=${sessionForMount} cwd=${plan.cwd}`, + ); + if ( + await (deps.mountStorage ?? mountStorage)( + plan.cwd, + environment.mountCreds, + { + log: logger, + }, + ) + ) { + environment.mountedCwd = plan.cwd; + return true; + } + return false; + }; + let localDurableCwdEnotconnRemounts = 0; + const reSignAndRemountLocalCwd = async (): Promise => { + if (!sessionForMount || !runCred || plan.isDaytona) return false; + if ( + localDurableCwdEnotconnRemounts >= + LOCAL_DURABLE_CWD_ENOTCONN_REMOUNT_LIMIT + ) { + logger( + `local durable cwd ENOTCONN remount limit reached session=${sessionForMount} cwd=${plan.cwd}`, + ); + return false; + } + localDurableCwdEnotconnRemounts += 1; + logger( + `local durable cwd ENOTCONN session=${sessionForMount} cwd=${plan.cwd}; re-signing and remounting`, + ); + const fresh = await signMount(sessionForMount, { + apiBase: apiBase(), + authorization: runCred, + log: logger, + }); + if (!fresh) { + logger( + `local durable cwd re-sign returned no credentials session=${sessionForMount}`, + ); + return false; + } + environment.mountCreds = fresh; + return mountLocalDurableCwd("enotconn-retry"); + }; + const remountLocalCwdAfterRuntimeEnotconn = (event: unknown): void => { + if (plan.isDaytona || !environment.mountCreds || !environment.mountedCwd) + return; + if ( + environment.runtimeRemount || + !containsTransportEndpointDisconnected(event) + ) + return; + logger( + `local durable cwd ENOTCONN observed in ACP event session=${sessionForMount} cwd=${plan.cwd}; re-signing and remounting`, + ); + environment.runtimeRemount = reSignAndRemountLocalCwd().catch((err) => { + logger( + `local durable cwd runtime remount failed session=${sessionForMount}: ${conciseError(err, plan.harness)}`, + ); + return false; + }); + }; + + try { + // Persist events in-process so a follow-up turn can resume by session id. + const persist = + deps.createPersist?.() ?? new InMemorySessionPersistDriver(); + const startSandboxAgent = + deps.startSandboxAgent ?? + ((options: Parameters[0]) => + SandboxAgent.start(options)); + const sandboxProvider = (deps.buildSandboxProvider ?? buildSandboxProvider)( + plan.sandboxId, + env, + binaryPath, + piExtEnv, + plan.secrets, + plan.sandboxPermission, + ); + const startOptions = { + sandbox: sandboxProvider, + persist, + // Propagate caller cancellation (a client disconnect on the streaming HTTP edge) so an + // in-flight run aborts instead of finishing unobserved. `destroy` still disposes. + ...(signal ? { signal } : {}), + // Long-timeout undici dispatcher so a paused HITL turn is not reaped by undici's default + // headersTimeout; Daytona additionally carries the per-sandbox auth cookie. + fetch: plan.isDaytona + ? (deps.createCookieFetch ?? createCookieFetch)() + : (deps.createAcpFetch ?? createAcpFetch)(), + }; + // A stored sandbox id is trusted: reconnect it by id and let reconnect converge its network + // policy to this run's plan. Any reconnect failure falls through to a fresh create. Snapshot + // and image drift are accepted as per-conversation version pinning, not grounds for a rebuild. + const storedSandboxPointer = + plan.isDaytona && sessionForMount && runCred + ? await (deps.readStoredSandboxPointer ?? readStoredSandboxPointer)( + sessionForMount, + { authorization: runCred, log: logger }, + ) + : undefined; + if (storedSandboxPointer) { + const sandboxStartStartedAt = Date.now(); + try { + environment.sandbox = await startSandboxAgent({ + ...startOptions, + sandboxId: storedSandboxPointer.sandboxId, + }); + logger( + `reconnected sandbox=${storedSandboxPointer.sandboxId} session=${sessionForMount}`, + ); + } catch (err) { + logger( + `reconnect failed sandbox=${storedSandboxPointer.sandboxId}, creating fresh: ${conciseError(err, plan.harness)}`, + ); + if ( + err instanceof DaytonaReconnectTerminalError && + sessionForMount && + runCred + ) { + // The post-hydrate write later in acquire is authoritative. This clear only prevents + // repeated doomed reconnects if acquire fails before reaching that write. Hydrate + // first: after a runner restart the in-memory store is behind the durable + // latest_turn_index, and an unhydrated guard token would be rejected as stale. + await ( + deps.hydrateHarnessSessionFromDurable ?? + hydrateHarnessSessionFromDurable + )( + sessionForMount, + plan.harness, + deps.sessionContinuityStore ?? sessionContinuityStore, + { authorization: runCred, log: logger }, + ); + await (deps.clearSandboxPointer ?? clearSandboxPointer)( + sessionForMount, + nextTurnIndex( + sessionForMount, + deps.sessionContinuityStore ?? sessionContinuityStore, + ), + { authorization: runCred, log: logger }, + ); + } + } finally { + timingLog("sandbox_start", sandboxStartStartedAt, " mode=reconnect"); + } + } + if (!environment.sandbox) { + const sandboxStartStartedAt = Date.now(); + try { + environment.sandbox = await startSandboxAgent(startOptions); + } finally { + timingLog("sandbox_start", sandboxStartStartedAt, " mode=create"); + } + } + environment.resumable = Boolean(plan.isDaytona && sessionForMount); + // Track the live handle so a shutdown signal handler can delete it if `destroy` is skipped by + // a process KILL; removed in `destroy` on every normal exit so it is never double-deleted. + if (environment.sandbox) inFlightSandboxes.add(environment); + + // On Daytona, push the harness login, the extension, and AGENTS.md into the remote sandbox. + // For a non-Pi harness with executable tools, also push the in-sandbox stdio MCP shim + // assets (bundle + public-specs file): a non-Pi harness in the sandbox cannot reach the + // runner-loopback HTTP MCP channel, so the harness's ACP adapter spawns the uploaded shim + // as the internal stdio MCP server instead. Uploaded unconditionally for non-Pi (the + // capability probe runs later; a harness that turns out to lack MCP fails loud in + // `assertRequiredCapabilities` below). Pi delivers via its extension; local non-Pi uses + // the loopback HTTP channel — neither needs this. The upload helper THROWS when the shim + // cannot be delivered (fail loud — this path requires it). + let internalToolMcp: ToolMcpAssets | undefined; + if (plan.isDaytona) { + await (deps.prepareDaytonaPiAssets ?? prepareDaytonaPiAssets)({ + sandbox: environment.sandbox, + plan, + log: logger, + }); + if (!plan.isPi && plan.executableToolSpecs.length > 0) { + internalToolMcp = await ( + deps.uploadToolMcpAssets ?? uploadToolMcpAssets + )( + environment.sandbox, + plan.toolMcpDir, + advertisedToolSpecs(plan.executableToolSpecs), + logger, + ); + } + } + + // Durable cwd: mount BEFORE createSession (so the session opens inside it) and BEFORE + // workspace materialization (so AGENTS.md, harness files, and skills land in the durable + // prefix instead of being hidden under the FUSE mount). + if (environment.mountCreds && !plan.isDaytona) { + await mountLocalDurableCwd("initial"); + } + if (environment.mountCreds && plan.isDaytona) { + const mountsStartedAt = Date.now(); + try { + // Mount against the store's own endpoint when the sandbox can reach it (public S3); fall + // back to the tunnel only for an in-network store. No tunnel + in-network store => skip. + const storeEndpoint = environment.mountCreds.endpoint; + const endpoint = storeReachableFromSandbox(storeEndpoint) + ? undefined + : ((await (deps.discoverTunnelEndpoint ?? discoverTunnelEndpoint)({ + log: logger, + })) ?? undefined); + const canMount = storeReachableFromSandbox(storeEndpoint) || !!endpoint; + if ( + canMount && + (await (deps.mountStorageRemote ?? mountStorageRemote)( + environment.sandbox, + plan.cwd, + environment.mountCreds, + { + endpoint, + log: logger, + }, + )) + ) { + logger(`remote durable cwd active for session=${sessionForMount}`); + } + // Per-harness session/transcript-dir mounts, remote-only by construction (this whole + // branch is `plan.isDaytona`) — local runs never reach here, so they stay mount-free/ + // byte-identical. Opt-out via env, default on wherever a durable cwd mount is active (no + // separate credential/session-id path from the cwd mount). + if ( + canMount && + sessionForMount && + runCred && + process.env.AGENTA_SESSION_HARNESS_MOUNTS !== "false" + ) { + const dirs = harnessSessionMounts( + plan.acpAgent, + "/home/sandbox", + DAYTONA_PI_DIR, + ); + await (deps.mountHarnessSessionDirs ?? mountHarnessSessionDirs)( + environment.sandbox, + sessionForMount, + dirs, + endpoint, + { + apiBase: apiBase(), + authorization: runCred, + log: logger, + }, + ); + } + } finally { + timingLog("mounts", mountsStartedAt); + } + } + + const prepareWorkspaceStartedAt = Date.now(); + try { + environment.workspace = await (deps.prepareWorkspace ?? prepareWorkspace)( + { + sandbox: environment.sandbox, + plan, + log: logger, + }, + ); + } catch (err) { + if ( + !plan.isDaytona && + environment.mountCreds && + isTransportEndpointDisconnected(err) && + (await reSignAndRemountLocalCwd()) + ) { + logger( + `retrying workspace preparation after local durable cwd remount`, + ); + environment.workspace = await ( + deps.prepareWorkspace ?? prepareWorkspace + )({ + sandbox: environment.sandbox, + plan, + log: logger, + }); + } else { + throw err; + } + } finally { + timingLog("prepare_workspace", prepareWorkspaceStartedAt); + } + + // Sandbox-start invariant: `startSandboxAgent` must hand back a usable handle. + assert( + environment.sandbox && + typeof environment.sandbox.createSession === "function", + `sandbox provider '${plan.sandboxId}' returned no usable sandbox handle`, + ); + + // Probe what this harness supports and branch on capabilities, not on the harness name. + const probeCapabilitiesStartedAt = Date.now(); + let probed; + try { + probed = await (deps.probeCapabilities ?? probeCapabilities)( + environment.sandbox, + plan.acpAgent, + ); + } finally { + timingLog("probe_capabilities", probeCapabilitiesStartedAt); + } + const capabilities = probed.capabilities; + environment.capabilities = capabilities; + + // Fail loud (A7): a run that REQUIRES a capability the harness lacks errors specifically + // rather than silently dropping the behavior. + assertRequiredCapabilities({ + harness: plan.harness, + isPi: plan.isPi, + probed, + toolSpecs: plan.toolSpecs, + log: logger, + }); + + const sessionMcp = await buildSessionMcpServers({ + isPi: plan.isPi, + capabilities, + harness: plan.harness, + isDaytona: plan.isDaytona, + toolSpecs: plan.toolSpecs, + userMcpServers: request.mcpServers, + relayDir: plan.relayDir, + clientToolRelay: deferredClientToolRelay, + signal: mcpAbort.signal, + // The uploaded in-sandbox stdio MCP shim assets, set only on Daytona + non-Pi + + // executable-tools; advertises the gateway tools the loopback channel cannot reach + // from inside the sandbox. No server to close for this entry (the harness owns the + // shim process), so `sessionMcp.close` semantics are unchanged. + internalToolMcp, + log: logger, + }); + // Close the internal gateway-tool MCP server (if one started) when the session is destroyed. + environment.closeToolMcp = sessionMcp.close; + + // If this harness authored the conversation's most recent turn (staleness-guarded) and we + // still remember its native `agentSessionId`, seed the fresh persist driver with a synthetic + // record and resume-by-id so the patched `resumeSession` reaches `session/load` instead of + // `session/new`. Any failure inside `resumeSession` already degrades to a plain new session + // internally (the patch's own `catch {}` around `loadRemoteSession`), so this call is safe to + // attempt unconditionally whenever we have an eligible id — worst case it is exactly today's + // cold `createSession`. + const continuitySessionKey = request.sessionId?.trim(); + const continuityStore = + deps.sessionContinuityStore ?? sessionContinuityStore; + // Seed the in-memory store from the durable row before consulting it, so a resume after a + // runner restart (in-memory map lost) still sees the prior turn's eligibility. No-op (and + // cheap) when the store already has a live in-process record. + if (continuitySessionKey && runCred) { + await ( + deps.hydrateHarnessSessionFromDurable ?? + hydrateHarnessSessionFromDurable + )(continuitySessionKey, plan.harness, continuityStore, { + authorization: runCred, + log: logger, + }); + } + const priorAgentSessionId = continuitySessionKey + ? eligibleAgentSessionId( + continuitySessionKey, + plan.harness, + continuityStore, + ) + : undefined; + const localSessionId = continuitySessionKey + ? `${continuitySessionKey}:${plan.harness}` + : undefined; + // The index THIS turn will occupy once it completes: recorded post-turn against the SAME + // index read here, so a turn that authors turn N leaves the store agreeing with itself. + environment.continuityTurnIndex = continuitySessionKey + ? nextTurnIndex(continuitySessionKey, continuityStore) + : undefined; + // Daytona only: a local run must not overwrite a conversation's remote pointer (switching + // sandboxes mid-conversation would strand the parked Daytona instance). + if (plan.isDaytona && sessionForMount && runCred) { + const liveSandboxId = environment.sandbox?.sandboxId ?? plan.sandboxId; + const pointerWriteOutcome = await ( + deps.writeSandboxPointer ?? writeSandboxPointer + )( + sessionForMount, + { + sandboxId: liveSandboxId, + turnIndex: environment.continuityTurnIndex ?? 0, + }, + { authorization: runCred, log: logger }, + ); + logger( + `sandbox pointer write ${pointerWriteOutcome} session=${sessionForMount} sandbox=${liveSandboxId}`, + ); + } + let loadedFromContinuity = false; + if (priorAgentSessionId && localSessionId) { + await persist.updateSession({ + id: localSessionId, + agent: plan.acpAgent, + agentSessionId: priorAgentSessionId, + lastConnectionId: "", + createdAt: Date.now(), + sessionInit: { cwd: plan.cwd, mcpServers: sessionMcp.servers }, + }); + const createSessionStartedAt = Date.now(); + try { + environment.session = + await environment.sandbox.resumeSession(localSessionId); + loadedFromContinuity = + environment.session.agentSessionId === priorAgentSessionId; + logger( + `[continuity] session/load attempted session=${continuitySessionKey} ` + + `harness=${plan.harness} loaded=${loadedFromContinuity}`, + ); + } catch (err) { + logger( + `[continuity] resumeSession failed, falling back to cold createSession: ` + + `${conciseError(err, plan.harness)}`, + ); + } finally { + timingLog("create_session", createSessionStartedAt, " mode=load"); + } + } + environment.loadedFromContinuity = loadedFromContinuity; + if (!environment.session) { + const createSessionStartedAt = Date.now(); + try { + environment.session = await environment.sandbox.createSession({ + ...(localSessionId ? { id: localSessionId } : {}), + agent: plan.acpAgent, + cwd: plan.cwd, + sessionInit: { cwd: plan.cwd, mcpServers: sessionMcp.servers }, + }); + } finally { + timingLog("create_session", createSessionStartedAt, " mode=create"); + } + } + environment.sessionId = resolveRunSessionId( + request, + environment.session.id, + ); + + // Resolve the model first: when the harness rejects the requested id and keeps its own + // default, `model` is undefined and the chat span is labelled "chat". + environment.model = await (deps.applyModel ?? applyModel)( + environment.session, + request.model, + logger, + { strict: strictModel }, + ); + + // Session-lifetime listeners: attach ONCE, each demuxing into the active turn's sink. They + // outlive any single turn, so the routing lives in dedicated non-throwing helpers below. + environment.session.onEvent((event: SessionEvent) => + routeSessionEventToActiveTurn( + environment, + remountLocalCwdAfterRuntimeEnotconn, + event, + ), + ); + environment.session.onPermissionRequest((req: SessionPermissionRequest) => + routePermissionRequestToActiveTurn(environment, req), + ); + + timingLog("acquire_total", acquireStartedAt); + return { ok: true, env: environment }; + } catch (err) { + const error = conciseError(err, plan.harness, request.provider); + // Mirror today's shared teardown: no otel exists yet during acquire, so there is no partial + // trace to flush — just run the incrementally-registered finalizers and surface the error. + await environment.destroy({ reason: "failed-turn" }); + return { ok: false, error }; + } +} + +import { +routePermissionRequestToActiveTurn, +routeSessionEventToActiveTurn, +} from "./session-events.ts"; +/** + * The cold, one-turn-per-environment entry (also the flag-off path). Acquire an environment, run + * one turn, then tear the environment down — exactly as the single `try/finally` did before the + * split, so behavior here is byte-identical to pre-keep-alive. + */ +/** + * Drop the harness's continuity record after a turn that did not complete. The harness may have + * written a partial turn into its native transcript, so a later `session/load` would resume a + * history the canonical request never sent. Dropping it falls back to cold replay. + */ +export function invalidateContinuity( + sessionId: string | undefined, + harness: string, + deps: SandboxAgentDeps, +): void { + if (!sessionId) return; + (deps.sessionContinuityStore ?? sessionContinuityStore).invalidate( + sessionId, + harness, + ); +} diff --git a/services/runner/src/engines/sandbox_agent/model.ts b/services/runner/src/engines/sandbox_agent/model.ts index b66b24c313..8d9b45343f 100644 --- a/services/runner/src/engines/sandbox_agent/model.ts +++ b/services/runner/src/engines/sandbox_agent/model.ts @@ -46,7 +46,10 @@ const stripContextHint = (id: string) => id.replace(/\[[^[\]]*\]$/, ""); * context) variant; it never falls back from a hinted request to a bare id, which would silently * shrink the context window. */ -export function pickModel(allowed: string[], wanted?: string): string | undefined { +export function pickModel( + allowed: string[], + wanted?: string, +): string | undefined { if (!wanted) return undefined; if (allowed.includes(wanted)) return wanted; const suffix = (id: string) => id.slice(id.indexOf("/") + 1); @@ -58,18 +61,39 @@ export function pickModel(allowed: string[], wanted?: string): string | undefine ); } +interface ModelOptionsReader { + getConfigOptions?: () => Promise; +} + +interface ModelSession extends ModelOptionsReader { + setModel(model: string): Promise; +} + +function selectableModelIds(options: unknown): string[] { + if (!Array.isArray(options)) return []; + const modelOption = options.find((option) => { + if (!option || typeof option !== "object") return false; + const record = option as Record; + return record.category === "model" || record.id === "model"; + }); + if (!modelOption || typeof modelOption !== "object") return []; + const choices = (modelOption as Record).options; + if (!Array.isArray(choices)) return []; + return choices.flatMap((choice) => { + if (!choice || typeof choice !== "object") return []; + const record = choice as Record; + const id = record.value ?? record.id; + return typeof id === "string" && id.length > 0 ? [id] : []; + }); +} + /** Enumerate the harness's selectable model ids from the session config options. */ -export async function allowedModels(session: any): Promise { +export async function allowedModels( + session: ModelOptionsReader, +): Promise { + if (!session.getConfigOptions) return []; try { - const options = await session.getConfigOptions(); - const modelOpt = (options ?? []).find( - (o: any) => o.category === "model" || o.id === "model", - ); - const choices = modelOpt?.options ?? []; - // pi-acp builds each choice as `{ value: model.modelId, name, description }` and sandbox-agent - // reads `entry.value`; older shapes used `id`. Read `value` first so this returns the real - // selectable ids (reading only `id` silently returned [] for pi-acp). - return choices.map((c: any) => c.value ?? c.id).filter(Boolean); + return selectableModelIds(await session.getConfigOptions()); } catch { return []; } @@ -77,7 +101,9 @@ export async function allowedModels(session: any): Promise { /** Parse the allowed model ids out of an UnsupportedSessionValueError message. */ export function allowedFromError(err: unknown): string[] { - const match = /Allowed values:\s*(.+?)\s*$/.exec(String((err as Error)?.message ?? err)); + const match = /Allowed values:\s*(.+?)\s*$/.exec( + String((err as Error)?.message ?? err), + ); if (!match) return []; return match[1] .split(",") @@ -97,7 +123,7 @@ export function allowedFromError(err: unknown): string[] { * `AGENTA_AGENT_MODEL_STRICT=false`). This is the F-007 fix. */ export async function applyModel( - session: any, + session: ModelSession, wanted?: string, log: Log = () => {}, options: { strict?: boolean } = {}, @@ -111,7 +137,9 @@ export async function applyModel( // The harness rejected the exact id. Resolve it against the harness's own selectable ids // (Pi exposes "openai-codex/gpt-5.5"; a caller passes a bare "gpt-5.5") and retry once. const allowed = allowedFromError(err); - const fallbackAllowed = allowed.length ? allowed : await allowedModels(session); + const fallbackAllowed = allowed.length + ? allowed + : await allowedModels(session); const match = pickModel(fallbackAllowed, wanted); if (match && match !== wanted) { try { @@ -122,9 +150,15 @@ export async function applyModel( } } if (strict) { - throw new ModelNotSettableError(wanted, fallbackAllowed, (err as Error).message); + throw new ModelNotSettableError( + wanted, + fallbackAllowed, + (err as Error).message, + ); } - log(`model '${wanted}' not settable (${(err as Error).message}); using harness default`); + log( + `model '${wanted}' not settable (${(err as Error).message}); using harness default`, + ); return undefined; } } diff --git a/services/runner/src/engines/sandbox_agent/mount.ts b/services/runner/src/engines/sandbox_agent/mount.ts index 428c8ad493..97278d6ad8 100644 --- a/services/runner/src/engines/sandbox_agent/mount.ts +++ b/services/runner/src/engines/sandbox_agent/mount.ts @@ -458,8 +458,8 @@ export async function discoverTunnelEndpoint( const https = tunnels.find( (t) => t.proto === "https" && !!t.public_url, )?.public_url; - const any = tunnels.find((t) => !!t.public_url)?.public_url; - return https ?? any ?? null; + const fallback = tunnels.find((t) => !!t.public_url)?.public_url; + return https ?? fallback ?? null; } catch (err) { log( `tunnel discovery failed: ${String(err instanceof Error ? err.message : err).slice(0, 160)}`, @@ -477,7 +477,7 @@ export interface SandboxExec { env?: Record; timeoutMs?: number; }) => Promise< - { exitCode?: number; stderr?: unknown; result?: unknown } | undefined + { exitCode?: number | null; stderr?: unknown; result?: unknown } | undefined >; } @@ -611,7 +611,9 @@ export async function mountStorageRemote( await unmountRemoteDeadMount(sandbox, cwd, log); return false; } - log(`remote mounted ${creds.bucket}:${creds.prefix} -> ${cwd} (verified alive)`); + log( + `remote mounted ${creds.bucket}:${creds.prefix} -> ${cwd} (verified alive)`, + ); return true; } catch (err) { log( @@ -667,7 +669,9 @@ export async function mountHarnessSessionDirs( dir.name, ); if (!creds) { - log(`harness session mount '${dir.name}' not signed — skipping ${dir.path}`); + log( + `harness session mount '${dir.name}' not signed — skipping ${dir.path}`, + ); continue; } await mountRemote(sandbox, dir.path, creds, { diff --git a/services/runner/src/engines/sandbox_agent/pi-assets.ts b/services/runner/src/engines/sandbox_agent/pi-assets.ts index 63d9340a61..b69316b196 100644 --- a/services/runner/src/engines/sandbox_agent/pi-assets.ts +++ b/services/runner/src/engines/sandbox_agent/pi-assets.ts @@ -17,6 +17,7 @@ import { advertisedToolSpecs } from "../../tools/public-spec.ts"; import type { MaterializedSkill } from "../skills.ts"; import { PKG_ROOT } from "./daemon.ts"; import type { RunPlan } from "./run-plan.ts"; +import type { SandboxFilePort } from "./sandbox-ports.ts"; type Log = (message: string) => void; @@ -156,7 +157,7 @@ export function writeSystemPromptLocal( /** Upload the system/append-system prompts into a Daytona sandbox's Pi agent dir. */ export async function uploadSystemPromptToSandbox( - sandbox: any, + sandbox: SandboxFilePort, agentDir: string, systemPrompt: string | undefined, appendSystemPrompt: string | undefined, @@ -183,7 +184,7 @@ export async function uploadSystemPromptToSandbox( /** Upload the extension bundle into a Daytona sandbox's Pi extensions dir. Best-effort. */ export async function uploadPiExtensionToSandbox( - sandbox: any, + sandbox: SandboxFilePort, agentDir: string, log: Log = () => {}, ): Promise { @@ -298,7 +299,7 @@ export function prepareLocalPiAssets({ /** Upload materialized skill dirs into a Daytona sandbox's Pi `skills/` user scope. */ export async function uploadSkillsToSandbox( - sandbox: any, + sandbox: SandboxFilePort, agentDir: string, skillDirs: MaterializedSkill[], log: Log = () => {}, @@ -318,7 +319,7 @@ export async function uploadSkillsToSandbox( /** Recursively upload a host directory tree into a sandbox path via the FS API. */ export async function uploadDirToSandbox( - sandbox: any, + sandbox: SandboxFilePort, srcDir: string, destDir: string, ): Promise { diff --git a/services/runner/src/engines/sandbox_agent/provider.ts b/services/runner/src/engines/sandbox_agent/provider.ts index c13adb40cc..681116acce 100644 --- a/services/runner/src/engines/sandbox_agent/provider.ts +++ b/services/runner/src/engines/sandbox_agent/provider.ts @@ -1,4 +1,5 @@ import { local } from "sandbox-agent/local"; +import type { SandboxAgentSpawnLogMode } from "sandbox-agent"; import type { SandboxPermission } from "../../protocol.ts"; import { daytonaEnvVars } from "./daytona.ts"; @@ -8,8 +9,8 @@ import { daytonaWithLifecycle } from "./daytona-provider.ts"; * Translate the Layer 2 network policy into Daytona create fields. Daytona enforces egress * at the sandbox boundary: `networkBlockAll` blocks all outbound, `networkAllowList` is a * COMMA-SEPARATED CIDR string (not an array). `mode: "on"` (or no policy) leaves both unset - * so the sandbox stays default-open. The create object is cast `as any` at the call site, so - * these pass through even though the daytona wrapper's create type does not surface them. + * so the sandbox stays default-open. The lifecycle wrapper accepts these provider fields and + * passes them to Daytona creation unchanged. * * `mode: "allowlist"` with an EMPTY list maps to `networkBlockAll` (block-all), not default-open: * "allow these zero ranges" is faithfully read as "allow nothing", and it keeps this mapping @@ -38,7 +39,10 @@ export function daytonaNetworkFields( export const DEFAULT_DAYTONA_AUTOSTOP_MINUTES = 15; export const DEFAULT_DAYTONA_AUTODELETE_MINUTES = 30; -function positiveMinutes(rawValue: string | undefined, fallback: number): number { +function positiveMinutes( + rawValue: string | undefined, + fallback: number, +): number { const parsed = Number(rawValue); if (!Number.isFinite(parsed) || parsed < 1) return fallback; return Math.floor(parsed); @@ -118,7 +122,7 @@ export function buildSandboxProvider( const image = process.env.DAYTONA_IMAGE; return daytonaWithLifecycle({ ...(image ? { image } : {}), - create: buildDaytonaCreate(piExtEnv, secrets, sandboxPermission) as any, + create: buildDaytonaCreate(piExtEnv, secrets, sandboxPermission), }); } @@ -136,6 +140,10 @@ export function buildSandboxProvider( } // local: spawn `sandbox-agent server` on this host with the daemon env merged in. - const logMode = (process.env.SANDBOX_AGENT_LOG_LEVEL ?? "silent") as any; + const configuredLogMode = process.env.SANDBOX_AGENT_LOG_LEVEL; + const logMode: SandboxAgentSpawnLogMode = + configuredLogMode === "inherit" || configuredLogMode === "pipe" + ? configuredLogMode + : "silent"; return local({ env, binaryPath, log: logMode }); } diff --git a/services/runner/src/engines/sandbox_agent/run-limits.ts b/services/runner/src/engines/sandbox_agent/run-limits.ts index 7afca86929..6079bcf3f9 100644 --- a/services/runner/src/engines/sandbox_agent/run-limits.ts +++ b/services/runner/src/engines/sandbox_agent/run-limits.ts @@ -1,3 +1,5 @@ +import type { AgentEvent } from "../../protocol.ts"; + /** * Time-based run limits: the runner had no deadline anywhere, so a wedged run (daemon up but * never engaging, or a harness that stops making progress mid-turn) held its sandbox/mount/socket @@ -76,7 +78,7 @@ export interface RunLimitsHandle { noteToolCallEnd(id: string): void; /** Wrap an `EmitEvent` sink so every event it sees also resets idle/TTFB — the one * observation point every harness's progress already flows through. */ - wrapEmit(emit: (event: any) => void): (event: any) => void; + wrapEmit(emit: (event: AgentEvent) => void): (event: AgentEvent) => void; /** The turn parked for human input: freeze every timer for good (the pause path owns the * turn's end from here; these deadlines must never re-fire on top of it). */ notePaused(): void; @@ -184,7 +186,7 @@ export function createRunLimits( // Every event is progress for idle/TTFB purposes; per-tool-call timers are driven // separately by noteToolCallStart/End (called from the raw ACP update handler, which // knows the harness's tool-call id before this typed event is even built). - return (event: any) => { + return (event: AgentEvent) => { noteProgress(); emit(event); }; diff --git a/services/runner/src/engines/sandbox_agent/run-turn.ts b/services/runner/src/engines/sandbox_agent/run-turn.ts new file mode 100644 index 0000000000..40c2ea00ad --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/run-turn.ts @@ -0,0 +1,599 @@ + + +import { +type PermissionReply +} from "sandbox-agent"; + +import { +PendingApprovalLatch, +permissionsFromRequest, +} from "../../permission-plan.ts"; +import { +resolvePromptText, +type AgentRunRequest, +type AgentRunResult, +type EmitEvent, +type ToolCallbackContext +} from "../../protocol.ts"; +import { +ApprovalResponder, +ApprovedExecutionGrants, +ConversationDecisions, +extractApprovalDecisions, +extractClientToolOutputs +} from "../../responder.ts"; +import { +buildWorkflowReferences, +createInteraction, +resolveInteraction, +} from "../../sessions/interactions.ts"; +import { +localRelayHost, +sandboxRelayHost, +startToolRelay, +type RelayExecutionGuard, +} from "../../tools/relay.ts"; +import { +createSandboxAgentOtel, +TOOL_NOT_EXECUTED_PAUSED, +} from "../../tracing/otel.ts"; +import { +attachPermissionResponder +} from "./acp-interactions.ts"; +import { +buildClientToolRelay +} from "./client-tools.ts"; +import { +invalidateContinuity, +} from "./environment.ts"; +import { conciseError } from "./errors.ts"; +import { +PAUSED, +PendingApprovalPauseController, +} from "./pause.ts"; +import { findSwallowedPiError } from "./pi-error.ts"; +import { buildRelayExecutionGuard } from "./relay-guard.ts"; +import { +createRunLimits, +resolveRunLimits, +} from "./run-limits.ts"; +import { +RUN_LIMIT_TRIPPED, +sendLastMessageOnly, +type CurrentTurn, +type ParkedApproval, +type RunTurnOptions, +type SessionEnvironment +} from "./runtime-contracts.ts"; +import { +runCredential, +serverPermissionsFromRequest, +shouldSuppressPausedToolCallUpdate, +} from "./runtime-policy.ts"; +import { +syncHarnessSessionDurable +} from "./session-continuity-durable.ts"; +import { +sessionContinuityStore +} from "./session-continuity.ts"; +import { priorMessages } from "./transcript.ts"; +import { resolveRunUsage } from "./usage.ts"; + +/** + * Run one turn against an acquired environment: start a fresh otel run, wire this turn's pause + * controller / latch / decisions / responder into `env.currentTurn`, restart the tool relay, + * send the prompt, resolve usage, and finish + flush the trace. It does NOT tear down the + * environment (the caller owns `env.destroy`). On a continuation the prompt is only the new user + * text (`buildTurnText` does not run); on a cold turn it is `plan.turnText`, exactly as before. + */ +export async function runTurn( + env: SessionEnvironment, + request: AgentRunRequest, + emit?: EmitEvent, + signal?: AbortSignal, + opts: RunTurnOptions = {}, +): Promise { + const { plan, logger, deps } = env; + const sessionId = env.sessionId; + // Reset the per-turn tool-call id record (the park folds the completed turn's ids into the + // expected next-history fingerprint). + env.lastTurnToolCallIds = []; + // Reset the per-turn approval-park bookkeeping. A fresh turn starts with no parked gate; this + // turn re-records it only if it pauses on a Claude ACP permission gate. (The dispatch has + // already captured any prior park into `opts.resume` before calling us.) + env.parkedApproval = undefined; + env.approvalGateCount = 0; + // Hoisted so the catch can flush a partial trace (mirroring the pre-split `otel?` handling — + // a createOtel throw must still return `{ ok: false }`, not propagate raw) and the finally can + // stop this turn's relay on EVERY exit path (a cleared sink must never orphan it). + let otel: ReturnType | undefined; + let activeTurn: CurrentTurn | undefined; + + // Time-based run deadlines (total/idle/TTFB/per-tool-call) for THIS turn: an idle/wedged harness + // has no deadline anywhere, so a silent or hung turn would hold its sandbox forever. Tripping a + // limit resolves the prompt race with `RUN_LIMIT_TRIPPED`, which ends the turn as an error so the + // caller's teardown (`runSandboxAgent`'s `finally`, or the keep-alive dispatch's evict-on-failure) + // reclaims the sandbox exactly as any other error does. Disposed in the `finally` on every path. + // A human pause retires the deadlines (`notePaused`): a HITL wait is legitimate, not a wedge. + const runLimits = (deps.createRunLimits ?? createRunLimits)( + (deps.resolveRunLimits ?? resolveRunLimits)(logger), + { log: logger }, + ); + let runLimitTrip: (() => void) | undefined; + let runLimitReason: string | undefined; + const runLimitTripped = new Promise((resolve) => { + runLimitTrip = resolve; + }); + runLimits.onTrip((reason) => { + runLimitReason = reason; + runLimitTrip?.(); + }); + + try { + const promptText = resolvePromptText(request); + // Cold: replay the full transcript (plan.turnText). Continuation or loaded: send only new text. + const turnText = sendLastMessageOnly(opts) ? promptText : plan.turnText; + + const run = (deps.createOtel ?? createSandboxAgentOtel)({ + harness: plan.harness, + model: env.model, + skills: plan.skillDirs.map((s) => s.name), + traceparent: request.context?.propagation?.traceparent, + baggage: request.context?.propagation?.baggage, + endpoint: request.telemetry?.exporters?.otlp?.endpoint, + authorization: request.telemetry?.exporters?.otlp?.headers?.authorization, + captureContent: request.telemetry?.capture?.content?.enabled, + emitSpans: !plan.isPi || plan.isDaytona, + // Every emitted event is a progress signal for the idle/TTFB deadlines (message/thought + // deltas, tool calls and results, usage, ...) — the one seam every harness's output flows + // through. Per-tool-call timers are driven separately from `handleUpdate` below. + emit: emit && runLimits.wrapEmit(emit), + }); + otel = run; + + run.start({ + prompt: promptText, + sessionId, + messages: [ + ...priorMessages(request), + { role: "user", content: promptText }, + ], + }); + + const pause = new PendingApprovalPauseController(() => { + // The sibling settle runs UNCONDITIONALLY, park mode or not: latch-loser tool calls + // announced before the winning gate can never execute this turn, and skipping the settle + // here would leave them as orphaned open parts whenever the dispatch later refuses the park + // (multi-gate, pool full) — `env.destroy()` does not re-run it. The exclusion keeps the + // gated (paused) call itself open, so the live resume is untouched. + run.settleOpenToolCalls( + (id) => pause.isPausedToolCall(id), + TOOL_NOT_EXECUTED_PAUSED, + ); + // Park mode: a parkable permission gate (Claude ACP or Pi ACP) recorded + // `env.parkedApproval` BEFORE firing this pause (the onUserApprovalGate hook runs before + // the single-pause latch). Keep the live session — the gated tool runs on the resume — so + // skip ONLY the mcpAbort and the destroySession. The teardown is not lost: the dispatch + // either parks the session or, if it decides not to (multi-gate, pool full), calls + // `env.destroy()` which runs them. A non-parkable pause (keep-alive off, client tool) + // never records `parkedApproval`, so it still tears down here exactly as today. + if (opts.approvalParkMode && env.parkedApproval) return; + // Abort any in-flight loopback `tools/call` (a paused Claude client tool) BEFORE the + // session teardown, so its handler cannot write a result after the turn ends. + env.mcpAbort.abort(); + env.sessionDestroyRequested = true; + return env.sandbox.destroySession?.(env.session.id); + }); + // A human pause resolves this signal exactly once, the moment the turn parks for input — the one + // place every pause path converges, so the one place to retire the run-limits deadlines for good. + void pause.signal.then(() => runLimits.notePaused()); + + // Publish this turn's sink so the session-lifetime listeners route into it. handleUpdate + // reproduces the old per-event routing (suppress paused frames, handleUpdate, pause re-sweep). + const turn: CurrentTurn = { + run, + pause, + toolRelay: undefined, + handleUpdate: (update) => { + // Per-tool-call deadline: starts on the announcement, ends on a terminal status. Tracked + // regardless of the pause-suppression below (a call already timed out must not linger just + // because a later sibling frame gets suppressed). + const rawFrame = update as { + sessionUpdate?: unknown; + toolCallId?: unknown; + status?: unknown; + }; + if (rawFrame?.sessionUpdate === "tool_call" && rawFrame.toolCallId) { + runLimits.noteToolCallStart(String(rawFrame.toolCallId)); + } else if ( + rawFrame?.sessionUpdate === "tool_call_update" && + rawFrame.toolCallId && + (rawFrame.status === "completed" || rawFrame.status === "failed") + ) { + runLimits.noteToolCallEnd(String(rawFrame.toolCallId)); + } + if (!shouldSuppressPausedToolCallUpdate(update, pause)) { + // Record the emitted tool-call ids (unique, first-seen order): the park folds them + // into the expected next-history fingerprint so a tool-using turn continues live. + const frame = update as { + sessionUpdate?: unknown; + toolCallId?: unknown; + }; + if ( + frame?.sessionUpdate === "tool_call" && + typeof frame.toolCallId === "string" && + frame.toolCallId && + !env.lastTurnToolCallIds.includes(frame.toolCallId) + ) { + env.lastTurnToolCallIds.push(frame.toolCallId); + } + run.handleUpdate(update); + // A sibling announced AFTER the pause won the latch can never execute; settle it + // immediately so the client never holds an orphaned part (idempotent re-sweep). + if (pause.active) { + run.settleOpenToolCalls( + (id) => pause.isPausedToolCall(id), + TOOL_NOT_EXECUTED_PAUSED, + ); + } + } + }, + onPermissionRequest: undefined, + }; + activeTurn = turn; + env.currentTurn = turn; + + const permissionPlan = permissionsFromRequest(request); + const storedDecisionMap = extractApprovalDecisions(request); + if (storedDecisionMap.size > 0) { + logger( + `[HITL] resume state: decisions=${JSON.stringify([...storedDecisionMap.keys()])}`, + ); + } + const decisions = new ConversationDecisions( + storedDecisionMap, + extractClientToolOutputs(request), + ); + const executionGrants = new ApprovedExecutionGrants(); + const latch = new PendingApprovalLatch(); + const responder = + deps.responderFactory?.(request) ?? + new ApprovalResponder(permissionPlan, decisions, logger); + // Every pause seeds the durable interactions plane, whichever gate paused. + const recordPendingInteraction = ( + token: string, + toolName: string | undefined, + toolArgs: unknown, + kind: "user_approval" | "client_tool" = "user_approval", + ): void => { + const cred = runCredential(request); + if (!cred) return; + const references = buildWorkflowReferences(request.runContext?.workflow); + if (!references?.workflow_revision) return; + void createInteraction( + sessionId, + request.turnId ?? "", + token, + kind, + { request: { tool: toolName ?? token, args: toolArgs }, references }, + () => cred, + ); + }; + // Transition the durable interaction row to resolved once its gate is answered. Used both by + // the cold decision-map path (via attachPermissionResponder) and the live approval resume, + // which answers the parked gate directly. It mirrors the cold path's ordering against + // `cancelStaleInteractions` (server.ts): that sweep cancels only PENDING gates of OTHER turns, + // and by resume time the human already marked this gate responded, so it is spared here too. + const resolveInteractionToken = (token: string): void => { + const cred = runCredential(request); + if (!cred) return; + if ( + !buildWorkflowReferences(request.runContext?.workflow) + ?.workflow_revision + ) + return; + void resolveInteraction(sessionId, token, () => cred); + }; + const serverPermissions = serverPermissionsFromRequest(request); + // Build the per-turn permission handler WITHOUT attaching to the live session: the + // session-lifetime `onPermissionRequest` (in acquireEnvironment) routes into it via + // `currentTurn`. A capturing shim reuses attachPermissionResponder unchanged; its + // respondPermission delegates to the real session. + attachPermissionResponder({ + session: { + onPermissionRequest: (handler: (req: unknown) => void) => { + turn.onPermissionRequest = handler; + }, + respondPermission: (id: string, reply: PermissionReply) => + env.session.respondPermission(id, reply), + }, + run, + responder, + latch, + serverPermissions, + log: logger, + onPause: () => pause.pause(), + onPausedToolCall: (id) => pause.markPausedToolCall(id), + onCreateInteraction: recordPendingInteraction, + onResolveInteraction: resolveInteractionToken, + // Pi runs only: presence of the specs map turns Pi gate envelope detection on AND is how + // the runner recovers specPermission/readOnlyHint (the envelope carries identity, never + // policy). Absent for Claude, so a title collision there keeps the base path. + piToolSpecsByName: plan.isPi + ? new Map( + plan.toolSpecs.map((spec) => [ + spec.name, + { + permission: spec.permission, + readOnly: spec.readOnly, + // callRef tools only: bound paths are runner-filled at execution, so the + // approval card and decision keys must not carry the model's values for them. + contextBindings: spec.callRef + ? spec.contextBindings + : undefined, + }, + ]), + ) + : undefined, + // A resolved custom-tool allow becomes an execution grant the relay guard consumes, so + // only a dialog-approved (or policy-allowed) call ever executes from the relay dir. + onPiGateAllowed: (info) => + executionGrants.grant(info.toolName, info.args), + // Record the parkable permission gate (only in keep-alive park mode) so the dispatch can + // resume it live. Fires per pending gate (before the latch) so a parallel gate is counted; + // the single-gate resume records only the FIRST gate's answer target. `info.gateType` names + // the plane (Claude ACP vs Pi ACP) so the resume answers on the right one. + onUserApprovalGate: opts.approvalParkMode + ? (info) => { + env.approvalGateCount += 1; + if ( + env.approvalGateCount === 1 && + info.permissionId && + info.toolCallId + ) { + env.parkedApproval = { + gateType: info.gateType, + permissionId: info.permissionId, + toolCallId: info.toolCallId, + toolName: info.toolName, + args: info.args, + interactionToken: info.interactionToken, + }; + } + } + : undefined, + }); + + // Resolve the ONE client-tool seam both delivery paths share. The correlation index is wired + // for Claude only — Pi's relay toolCallId is already exact. + env.clientToolRelayRef.current = buildClientToolRelay({ + responder, + run, + latch, + pause, + recordPendingInteraction, + toolCallIndex: plan.isPi ? undefined : env.toolCallIndex, + log: logger, + }); + + // EVERY harness gets the guard: the relay dir is sandbox-writable, so a forged + // `.req.json` proves nothing about any dialog having run, and this runner-side + // re-check is the only enforcement of the hard deny boundary against forged files. + // `allow` passes and `deny` refuses identically everywhere; `ask` splits by harness — + // Pi consumes a dialog-recorded execution grant (fail-closed parity with the in-sandbox + // confirm), while a non-Pi MCP harness (Claude) passes `ask` because its own harness + // enforces the ask dialog (the rendered `mcp__agenta-tools__` ask rules + the ACP + // permission flow) before a call reaches the shim. See buildRelayExecutionGuard for the + // stated residual (a forged file can still trigger an ask-tool without a dialog there). + const relayGuard: RelayExecutionGuard = buildRelayExecutionGuard({ + isPi: plan.isPi, + permissionPlan, + executionGrants, + }); + + if (plan.useToolRelay) { + turn.toolRelay = (deps.startToolRelay ?? startToolRelay)( + plan.isDaytona + ? (deps.sandboxRelayHost ?? sandboxRelayHost)(env.sandbox, { + log: logger, + }) + : (deps.localRelayHost ?? localRelayHost)(), + plan.relayDir, + plan.toolSpecs, + request.toolCallback as ToolCallbackContext | undefined, + request.runContext, + env.clientToolRelayRef.current, + relayGuard, + { log: logger }, + ); + // Ordering invariant: the relay's stale-file sweep must complete before the + // resume's respondPermission or the fresh prompt below can cause a legitimate + // request, so nothing legitimate can predate the sweep and be swallowed as + // stale. Optional-chained so a fake relay without `ready` is tolerated, and a + // sweep failure never kills the turn. + await turn.toolRelay?.ready?.catch?.(() => {}); + } + + // The prompt promise this turn races against the pause signal. A normal/continuation turn + // sends a fresh prompt; a live approval resume answers the parked gate on the SAME session and + // continues the ORIGINAL, still-pending prompt promise (the tool then runs with its original + // byte-exact args). Either way, on a HITL pause the prompt resolves cancelled or never + // resolves, and the pause signal ends the turn. + let promptPromise: Promise; + if (opts.resume) { + // The new (resume) turn owns streaming + tracing; the environment is already wired to route + // continued events into this turn's sink (env.currentTurn was set above). Seed this run's + // trace with the parked tool call so the completing `tool_call_update` closes it and the FE + // approval part flips to output-available even if the adapter re-announces nothing. Then + // answer the gate on the live session — the original prompt continues from here. + run.handleUpdate({ + sessionUpdate: "tool_call", + toolCallId: opts.resume.toolCallId, + title: opts.resume.toolName, + kind: opts.resume.toolName, + rawInput: opts.resume.args, + }); + promptPromise = Promise.resolve(opts.resume.promptPromise); + promptPromise.catch(() => {}); + // A parked Pi dialog gate resumes on a FRESH turn whose relay and grant ledger are new; + // grant the approved call here so the extension's execute record (written right after the + // confirm resolves) passes the relay guard. Claude resumes grant too — harmlessly, no + // guard consults it. + if (opts.resume.reply === "once") { + executionGrants.grant(opts.resume.toolName, opts.resume.args); + } + await env.session.respondPermission( + opts.resume.permissionId, + opts.resume.reply, + ); + // The gate is answered: resolve the durable interaction row (the parked pending row the cold + // path would otherwise resolve via its decision map). The fresh per-turn pause controller + // starts with an EMPTY pausedToolCallIds set, so the resumed call's `tool_call_update` frames + // are no longer suppressed and stream through — the "clear pausedToolCallIds on resume" step. + resolveInteractionToken(opts.resume.interactionToken); + logger( + `[keepalive] resume answered gate reply=${opts.resume.reply} tool=${opts.resume.toolName ?? "?"}`, + ); + } else { + promptPromise = Promise.resolve( + env.session.prompt([{ type: "text", text: turnText }]), + ); + promptPromise.catch(() => {}); + } + const raced = await Promise.race([ + promptPromise, + pause.signal.then(() => PAUSED), + runLimitTripped.then(() => RUN_LIMIT_TRIPPED), + ]); + // A tripped run-limit ends the turn as an error: throw into the shared catch below so the + // trace is flushed and the caller's teardown reclaims the (wedged) sandbox. + if (raced === RUN_LIMIT_TRIPPED) { + throw new Error(runLimitReason ?? "run limit tripped"); + } + const stopReason = + raced === PAUSED || pause.active + ? "paused" + : raced && typeof raced === "object" && "stopReason" in raced + ? String(raced.stopReason) + : undefined; + const result = raced === PAUSED ? undefined : raced; + // A parkable pause this turn: hand the still-pending prompt promise to the parked record so a + // later resume can await the same continuation. (Set after the race so `promptPromise` exists. + // The read is asserted because the onUserApprovalGate callback set the field via an async + // mutation TS's flow analysis cannot see, so it would otherwise narrow the reset to `never`.) + const parkedThisTurn = env.parkedApproval as ParkedApproval | undefined; + if (opts.approvalParkMode && pause.active && parkedThisTurn) { + parkedThisTurn.promptPromise = promptPromise; + } + await turn.toolRelay?.stop(); + logger(`prompt stopReason=${stopReason}`); + + const usage = await resolveRunUsage({ + sandbox: env.sandbox, + usageOutPath: plan.usageOutPath, + isDaytona: plan.isDaytona, + promptResult: result, + streamUsage: run.usage(), + }); + run.setUsage(usage); + + const swallowedPiError = + plan.isPi && + !plan.isDaytona && + !run.output().trim() && + !run.events().some((e) => e.type === "tool_call") + ? findSwallowedPiError( + env.runAgentDir ?? plan.sourcePiAgentDir, + plan.cwd, + ) + : undefined; + let swallowedError: string | undefined; + if (swallowedPiError) { + swallowedError = conciseError( + new Error(swallowedPiError), + plan.harness, + request.provider, + ); + run.recordError(swallowedError, request.provider); + run.emitEvent({ type: "error", message: swallowedError }); + } + + const output = run.finish(); + await run.flush(); + + if (swallowedError) { + // A failed turn may have left a partial turn in the native transcript: the prior record + // is no longer a faithful resume point. + invalidateContinuity(sessionId, plan.harness, deps); + return { ok: false, error: swallowedError }; + } + + // Capture this harness's native session id for the next turn's setup. Only on a turn that + // actually completed (not paused mid-turn — a park has not finished authoring the turn, so + // it must not be marked authoritative) and only when the harness surfaced one. + if ( + stopReason !== "paused" && + env.continuityTurnIndex !== undefined && + sessionId && + env.session?.agentSessionId + ) { + (deps.sessionContinuityStore ?? sessionContinuityStore).record( + sessionId, + plan.harness, + env.session.agentSessionId, + env.continuityTurnIndex, + ); + // Mirror the record durably so it survives a runner restart; fire-and-forget. + const syncCred = runCredential(request); + if (syncCred) { + void (deps.syncHarnessSessionDurable ?? syncHarnessSessionDurable)( + sessionId, + plan.harness, + env.session.agentSessionId, + env.continuityTurnIndex, + { authorization: syncCred, log: logger }, + ); + } + } else if (stopReason === "paused") { + // A pause stopped mid-turn, after the harness may have written a partial turn natively. + invalidateContinuity(sessionId, plan.harness, deps); + } + + return { + ok: true, + output, + messages: output ? [{ role: "assistant", content: output }] : [], + events: emit ? [] : run.events(), + usage, + stopReason, + capabilities: { + ...env.capabilities, + streamingDeltas: !!emit && env.capabilities.streamingDeltas, + }, + sessionId, + model: env.model ?? request.model, + traceId: run.traceId(), + } as AgentRunResult; + } catch (err) { + const error = conciseError(err, plan.harness, request.provider); + otel?.recordError(error, request.provider); + otel?.emitEvent({ type: "error", message: error }); + // An aborted turn may have left a partial turn in the native transcript. + invalidateContinuity(sessionId, plan.harness, deps); + // finish() must not throw uncaught — tracing must not mask the run error. + try { + otel?.finish(); + } catch {} + await otel?.flush().catch(() => {}); + return { ok: false, error }; + } finally { + // Release every run-limits timer (idempotent, never re-arms on a late event) on EVERY path. + runLimits.dispose(); + // This turn owns its relay: stop it on EVERY exit path (the happy path already stopped it + // after the prompt; stop is safe to repeat, matching the old finally). Null it afterwards so + // a later `destroy()` — possibly after the dispatch cleared the sink — cannot double-stop or + // orphan it. + await activeTurn?.toolRelay?.stop().catch(() => {}); + if (activeTurn) activeTurn.toolRelay = undefined; + } +} diff --git a/services/runner/src/engines/sandbox_agent/runtime-contracts.ts b/services/runner/src/engines/sandbox_agent/runtime-contracts.ts new file mode 100644 index 0000000000..69a1deea80 --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/runtime-contracts.ts @@ -0,0 +1,295 @@ + + +import { +InMemorySessionPersistDriver, +SandboxAgent +} from "sandbox-agent"; + +import { +type AgentRunRequest, +type HarnessCapabilities +} from "../../protocol.ts"; +import { +type Responder +} from "../../responder.ts"; +import type { ClientToolRelay } from "../../tools/client-tool-relay.ts"; +import { +localRelayHost, +sandboxRelayHost, +startToolRelay +} from "../../tools/relay.ts"; +import { +createSandboxAgentOtel +} from "../../tracing/otel.ts"; +import { createAcpFetch } from "./acp-fetch.ts"; +import { +type ParkedApprovalGateType +} from "./acp-interactions.ts"; +import { +probeCapabilities +} from "./capabilities.ts"; +import { +createToolCallCorrelationIndex +} from "./client-tools.ts"; +import { buildDaemonEnv,resolveDaemonBinary } from "./daemon.ts"; +import { +createCookieFetch, +prepareDaytonaPiAssets +} from "./daytona.ts"; +import { applyModel } from "./model.ts"; +import { +discoverTunnelEndpoint, +mountHarnessSessionDirs, +mountStorage, +mountStorageRemote, +signSessionMountCredentials, +unmountStorage, +type MountCredentials +} from "./mount.ts"; +import { +PendingApprovalPauseController +} from "./pause.ts"; +import { buildSandboxProvider } from "./provider.ts"; +import { +createRunLimits, +resolveRunLimits, +} from "./run-limits.ts"; +import { +type BuildRunPlanDeps, +type RunPlan +} from "./run-plan.ts"; +import { +clearSandboxPointer, +readStoredSandboxPointer, +writeSandboxPointer, +} from "./sandbox-reconnect.ts"; +import { +hydrateHarnessSessionFromDurable, +syncHarnessSessionDurable, +} from "./session-continuity-durable.ts"; +import { +type SessionContinuityStore +} from "./session-continuity.ts"; +import { +type TeardownReason +} from "./teardown.ts"; +import { +uploadToolMcpAssets +} from "./tool-mcp-assets.ts"; +import { prepareWorkspace } from "./workspace.ts"; + +type Log = (message: string) => void; + +export interface SandboxAgentDeps extends BuildRunPlanDeps { + startSandboxAgent?: typeof SandboxAgent.start; + createPersist?: () => InMemorySessionPersistDriver; + createOtel?: typeof createSandboxAgentOtel; + buildDaemonEnv?: typeof buildDaemonEnv; + resolveDaemonBinary?: typeof resolveDaemonBinary; + buildSandboxProvider?: typeof buildSandboxProvider; + createCookieFetch?: typeof createCookieFetch; + createAcpFetch?: typeof createAcpFetch; + prepareWorkspace?: typeof prepareWorkspace; + prepareDaytonaPiAssets?: typeof prepareDaytonaPiAssets; + uploadToolMcpAssets?: typeof uploadToolMcpAssets; + probeCapabilities?: typeof probeCapabilities; + applyModel?: typeof applyModel; + startToolRelay?: typeof startToolRelay; + localRelayHost?: typeof localRelayHost; + sandboxRelayHost?: typeof sandboxRelayHost; + signSessionMountCredentials?: typeof signSessionMountCredentials; + mountStorage?: typeof mountStorage; + mountStorageRemote?: typeof mountStorageRemote; + unmountStorage?: typeof unmountStorage; + discoverTunnelEndpoint?: typeof discoverTunnelEndpoint; + /** Per-harness transcript mounts (remote only; see mount.ts). */ + mountHarnessSessionDirs?: typeof mountHarnessSessionDirs; + responderFactory?: (request: AgentRunRequest) => Responder; + resolveRunLimits?: typeof resolveRunLimits; + createRunLimits?: typeof createRunLimits; + /** Session-continuity store override (tests inject their own; default is the process singleton). */ + sessionContinuityStore?: SessionContinuityStore; + /** Durable read-back/write-forward of the continuity store (tests inject fakes). */ + hydrateHarnessSessionFromDurable?: typeof hydrateHarnessSessionFromDurable; + syncHarnessSessionDurable?: typeof syncHarnessSessionDurable; + /** Durable read/write of the sandbox pointer, for the remote reconnect ladder. */ + readStoredSandboxPointer?: typeof readStoredSandboxPointer; + clearSandboxPointer?: typeof clearSandboxPointer; + writeSandboxPointer?: typeof writeSandboxPointer; + /** + * Resolve `{replicaId, ownerReplicaId}` for a session-owned local-sandbox run, so + * `acquireEnvironment` can fail loudly instead of silently cold-starting on a non-owner + * replica. The default claims the `owner` affinity key via the coordination plane and reads + * back the actual owner (`claimSessionOwnership`); tests inject their own. `authorization` is + * the run credential (the claim authenticates as the invoke caller). + */ + resolveLocalRunnerOwner?: ( + sessionId: string, + authorization: string, + ) => Promise<{ replicaId: string; ownerReplicaId: string | undefined }>; + log?: Log; +} + + +/** + * Race sentinel: a run-limits deadline (total/idle/TTFB/per-tool-call) tripped mid-turn. Distinct + * from `PAUSED` so the prompt race can tell a human pause (keep the session) from a wedge deadline + * (end the turn as an error, letting the caller's teardown reclaim the sandbox). + */ +export const RUN_LIMIT_TRIPPED = Symbol("run-limit-tripped"); + +/** + * The per-turn sink the session-lifetime listeners demux into. `runTurn` swaps a fresh one in + * at turn start (`env.currentTurn`) and the dispatch clears it at turn end. The `sandbox-agent` + * listener registries are plain Sets — an event with no listener is dropped and a permission + * request with no listener is CANCELLED — so the listeners stay attached for the session's whole + * life and route into whichever turn is active, with no detach/attach window between turns. + */ +export interface CurrentTurn { + run: ReturnType; + pause: PendingApprovalPauseController; + toolRelay?: { ready?: Promise; stop: () => Promise }; + /** Route a session/update for the active turn (suppress + handleUpdate + pause re-sweep). */ + handleUpdate: (update: unknown) => void; + /** Route a permission reverse-RPC for the active turn (built by attachPermissionResponder). */ + onPermissionRequest?: (req: unknown) => void; +} + +/** + * A permission gate that paused the turn and can be answered later on the SAME live session. + * Recorded for a Claude ACP permission gate (keep-alive slice 2) or a Pi ACP permission gate + * (Pi approval parking: the gate rides the extension's `ctx.ui.confirm` onto the same ACP + * permission plane). NOT recorded for a client-tool MCP pause — that cannot be answered across + * a turn boundary and stays on the cold path. Existence of this record is what makes the + * dispatch park a paused session in `awaiting_approval` instead of tearing it down. + */ +export interface ParkedApproval { + /** Which gate paused; the dispatch resumes only a recognized type and treats others as cold. */ + gateType: ParkedApprovalGateType; + /** The ACP permission-request id, answered later via `session.respondPermission`. */ + permissionId: string; + /** The gated tool call's id — matched against the incoming approval envelope's toolCallId. */ + toolCallId: string; + /** The gated tool name (logging + the durable interaction row); never its args, in logs. */ + toolName: string | undefined; + /** The gated call's original args, used to seed the resume turn's trace/egress tool span. */ + args: unknown; + /** The durable interaction row token, resolved on the answer via the onResolveInteraction hook. */ + interactionToken: string; + /** The held original `prompt()` promise; the resume awaits it after `respondPermission`. */ + promptPromise?: Promise; +} + +/** Answer a parked Claude ACP permission gate on the live session (the keep-alive resume input). */ +export interface ResumeApprovalInput { + permissionId: string; + reply: "once" | "reject"; + toolCallId: string; + toolName: string | undefined; + args: unknown; + interactionToken: string; + promptPromise?: Promise; +} + +/** Per-turn options for `runTurn`. Absent (flag off / cold) means today's byte-identical path. */ +export interface RunTurnOptions { + /** A live continuation: send only the new user text instead of the full cold transcript. */ + continuation?: boolean; + /** + * The session was rehydrated via `session/load` (the patched `resumeSession`), so the harness + * already holds the prior turns natively. Like `continuation`, the prompt is only the new user + * text; `buildTurnText` must not run. Distinct field from `continuation` because the two arrive + * through different acquire paths (live pool checkout vs a fresh cold acquire that loaded an + * old session) — `runTurn` treats them identically for the text-selection decision. + */ + loaded?: boolean; + /** + * Keep-alive approval park mode: on a Claude ACP permission gate the pause keeps the session + * alive (no settle/abort/destroy) so a later resume can answer it. A non-parkable pause (Pi + * relay, client tool) still tears down exactly as today, so this is safe to set on any eligible + * keep-alive turn. + */ + approvalParkMode?: boolean; + /** A live approval resume: answer the parked gate and stream the continued prompt's events. */ + resume?: ResumeApprovalInput; +} + +/** + * Send only the new user text (not the full cold transcript) when the harness already holds the + * prior turns: a live continuation, or a session rehydrated via `session/load`. `runTurn` calls + * this, so a test that pins it pins the shipped decision. + */ +export function sendLastMessageOnly(opts: RunTurnOptions): boolean { + return Boolean(opts.continuation || opts.loaded); +} + +/** + * A session-scoped environment that can serve many turns. Everything expensive to build lives + * here (sandbox, session, internal tool-MCP server, mounted cwd, relay/temp dirs); `destroy()` + * is the one complete idempotent teardown the pool, the shutdown handler, and the cold path all + * call. Per-turn state rides `currentTurn`, swapped in by `runTurn`. + */ +export interface SessionEnvironment { + plan: RunPlan; + logger: Log; + deps: SandboxAgentDeps; + sandbox: any; + session: any; + sessionId: string; + model: string | undefined; + capabilities: HarnessCapabilities; + strictModel: boolean; + toolCallIndex: ReturnType; + /** The current turn's client-tool relay, read by the deferred ref baked into the MCP server. */ + clientToolRelayRef: { current?: ClientToolRelay }; + mcpAbort: AbortController; + runAgentDir: string | undefined; + otlpAuthFilePath: string | undefined; + mountCreds: MountCredentials | null; + /** The mount's owning project id (keep-alive pool key FALLBACK scope, preferred is + * `runContext.project.id`); undefined when there is no mount. */ + mountProjectId?: string; + /** This acquire resumed the harness's native session via `session/load` (not cold). */ + loadedFromContinuity: boolean; + /** A remote, session-owned run whose sandbox can be parked (warm) rather than deleted at end. */ + resumable: boolean; + /** The conversation turn index this acquire's continuity record was read/written at. */ + continuityTurnIndex: number | undefined; + // Mutable teardown/turn state shared across acquire, runTurn, and destroy. + sessionDestroyRequested: boolean; + mountedCwd: string | undefined; + durableCwdSafeToDelete: boolean; + workspace: { cleanup: () => Promise } | undefined; + runtimeRemount: Promise | undefined; + closeToolMcp: (() => Promise) | undefined; + currentTurn?: CurrentTurn; + /** + * The unique ACP tool-call ids the LAST completed turn emitted (reset at each turn start). + * The keep-alive dispatch folds them into the expected next-history fingerprint at park time, + * so a tool-using turn still matches its own continuation (the FE keeps assistant tool parts). + */ + lastTurnToolCallIds: string[]; + /** + * The Claude ACP permission gate the LAST turn paused on, or undefined. Set only for a harness + * ACP permission gate, reset at each turn start; the dispatch reads it after a paused turn to + * decide whether to park in `awaiting_approval` and, on the next request, how to resume. + */ + parkedApproval?: ParkedApproval; + /** + * How many Claude ACP permission gates resolved to pendingApproval THIS turn (reset at turn + * start). More than one means parallel gates the single-gate resume cannot answer, so the + * dispatch does not park (tears down cold as today). + */ + approvalGateCount: number; + destroyed: boolean; + /** Complete, idempotent teardown selected from the typed teardown reason. */ + destroy: (opts?: { reason?: TeardownReason }) => Promise; + /** End the active turn: clear the current-turn sink (called before a park). */ + clearTurn: () => void; +} + +export type AcquireEnvironmentResult = + | { ok: true; env: SessionEnvironment } + | { ok: false; error: string }; + + diff --git a/services/runner/src/engines/sandbox_agent/runtime-policy.ts b/services/runner/src/engines/sandbox_agent/runtime-policy.ts new file mode 100644 index 0000000000..a948c9ac5e --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/runtime-policy.ts @@ -0,0 +1,174 @@ + + + +import { +type AgentRunRequest, +type ToolPermission +} from "../../protocol.ts"; +import { claimSessionOwnership,REPLICA_ID } from "../../sessions/alive.ts"; +import { +PendingApprovalPauseController +} from "./pause.ts"; + +type Log = (message: string) => void; + +/** Extract the run credential from the OTLP export headers (initial value, constant for the run). */ +export function runCredential(request: AgentRunRequest): string { + const headers = (request.telemetry?.exporters?.otlp?.headers ?? {}) as Record< + string, + string + >; + return (headers["authorization"] ?? headers["Authorization"] ?? "").trim(); +} + +export function serverPermissionsFromRequest( + request: AgentRunRequest, +): ReadonlyMap { + const permissions = new Map(); + for (const server of request.mcpServers ?? []) { + if (server.permission !== undefined) { + permissions.set(server.name, server.permission); + } + } + return permissions; +} + + +export function shouldSuppressPausedToolCallUpdate( + update: unknown, + pause: PendingApprovalPauseController, +): boolean { + const frame = update as + | { sessionUpdate?: unknown; toolCallId?: unknown } + | undefined; + const kind = frame?.sessionUpdate; + if (kind !== "tool_call" && kind !== "tool_call_update") return false; + const toolCallId = + typeof frame?.toolCallId === "string" ? frame.toolCallId : undefined; + return pause.isPausedToolCall(toolCallId); +} + +const CLAUDE_STRICT_DEPLOYMENTS = new Set([ + "custom", + "bedrock", + "vertex", + "vertex_ai", +]); + +export function applyClaudeConnectionEnv( + env: Record, + request: AgentRunRequest, + acpAgent: string, + logger: Log, +): void { + if (acpAgent !== "claude") return; + + // Disable the Claude Agent SDK's Tool-Search feature for every Claude run. The bundled + // SDK defaults Tool-Search ON, which makes Claude DEFER the `agenta-tools` MCP tools and + // call them before their `inputSchema` is loaded — so it emits an empty `input: {}` and + // tools-with-args (reference workflows, commit_revision) never receive their arguments. + // Our tool count is small, so deferral buys nothing and only strips the schema. The SDK + // treats only `false`/`0`/`no`/`off` as off, so the string must be "false" (not "0"/"100"). + // This is applied after `buildDaemonEnv`'s clear and is not in `KNOWN_PROVIDER_ENV_VARS`, + // so it is never stripped, and it reaches the Daytona sandbox like `ANTHROPIC_BASE_URL`. + env.ENABLE_TOOL_SEARCH = "false"; + + const deployment = request.deployment; + const selectedModel = request.model; + const baseUrl = request.endpoint?.baseUrl; + if (baseUrl) { + env.ANTHROPIC_BASE_URL = baseUrl; + logger(`claude base_url: ${baseUrl}`); + } + + if (deployment === "bedrock") { + env.CLAUDE_CODE_USE_BEDROCK = "1"; + const region = request.endpoint?.region; + if (region) { + env.AWS_REGION = region; + env.AWS_DEFAULT_REGION ??= region; + } + } else if (deployment === "vertex" || deployment === "vertex_ai") { + env.CLAUDE_CODE_USE_VERTEX = "1"; + } + + if ( + selectedModel && + (baseUrl || (deployment && CLAUDE_STRICT_DEPLOYMENTS.has(deployment))) + ) { + env.ANTHROPIC_MODEL = selectedModel; + env.ANTHROPIC_CUSTOM_MODEL_OPTION = selectedModel; + logger( + `claude model=${selectedModel} deployment=${deployment ?? ""}`, + ); + } +} + +/** + * Whether a requested-but-unsettable model fails the run (F-007). Strict by default on every + * harness path: a user who picks a model either runs that model or sees a loud error, never a + * silent (often pricier) fallback to the harness default. `AGENTA_AGENT_MODEL_STRICT=false` is + * the explicit opt-out that restores the legacy warn-and-fallback behavior. A run that requests + * no model is unaffected either way — it keeps the harness default. + */ +export function modelResolutionStrict(): boolean { + return process.env.AGENTA_AGENT_MODEL_STRICT !== "false"; +} + +export async function defaultResolveLocalRunnerOwner( + sessionId: string, + authorization: string, +): Promise<{ replicaId: string; ownerReplicaId: string | undefined }> { + // No credential ⇒ the claim would 401; treat as "no known owner" (pass), never worse than today. + if (!authorization) { + return { replicaId: REPLICA_ID, ownerReplicaId: undefined }; + } + return claimSessionOwnership(sessionId, authorization); +} + +export function isTransportEndpointDisconnected(err: unknown): boolean { + const message = String(err instanceof Error ? err.message : err); + const code = + typeof err === "object" && err !== null && "code" in err + ? String((err as { code?: unknown }).code) + : ""; + return ( + code === "ENOTCONN" || + message.includes("ENOTCONN") || + message.includes("Transport endpoint is not connected") + ); +} + +export function containsTransportEndpointDisconnected(value: unknown): boolean { + const seen = new Set(); + + const visit = (current: unknown): boolean => { + if (typeof current === "string") { + return isTransportEndpointDisconnected(current); + } + if (current instanceof Error) { + return isTransportEndpointDisconnected(current); + } + if (!current || typeof current !== "object") { + return false; + } + if (seen.has(current)) { + return false; + } + seen.add(current); + + const code = + "code" in current ? String((current as { code?: unknown }).code) : ""; + if (code === "ENOTCONN") { + return true; + } + + if (Array.isArray(current)) { + return current.some(visit); + } + return Object.values(current as Record).some(visit); + }; + + return visit(value); +} + diff --git a/services/runner/src/engines/sandbox_agent/sandbox-ports.ts b/services/runner/src/engines/sandbox_agent/sandbox-ports.ts new file mode 100644 index 0000000000..7b2acd18dc --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/sandbox-ports.ts @@ -0,0 +1,24 @@ +/** Narrow structural ports shared by helpers that operate on a remote sandbox. */ +export interface SandboxDirectoryPort { + readonly sandboxId?: string; + mkdirFs(input: { path: string }): Promise; +} + +export interface SandboxFilePort extends SandboxDirectoryPort { + writeFsFile(input: { path: string }, body: string): Promise; +} + +export interface SandboxProcessPort extends SandboxDirectoryPort { + runProcess(input: { + command: string; + args?: string[]; + cwd?: string; + env?: Record; + timeoutMs?: number; + }): Promise; +} + +export type SandboxAssetPort = SandboxFilePort & SandboxProcessPort; +export type SandboxWorkspacePort = Partial< + SandboxFilePort & SandboxProcessPort +>; diff --git a/services/runner/src/engines/sandbox_agent/session-continuity.ts b/services/runner/src/engines/sandbox_agent/session-continuity.ts index 684c00b539..c704ffcbad 100644 Binary files a/services/runner/src/engines/sandbox_agent/session-continuity.ts and b/services/runner/src/engines/sandbox_agent/session-continuity.ts differ diff --git a/services/runner/src/engines/sandbox_agent/session-events.ts b/services/runner/src/engines/sandbox_agent/session-events.ts new file mode 100644 index 0000000000..4b6a73f549 --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/session-events.ts @@ -0,0 +1,88 @@ +import type { SessionEvent, SessionPermissionRequest } from "sandbox-agent"; + +import { conciseError } from "./errors.ts"; +import type { SessionEnvironment } from "./runtime-contracts.ts"; + +/** + * Route one harness event into the active turn's sink. + * + * Data flow: the ACP session emits an event -> we demux it -> the active turn + * (`environment.currentTurn`) consumes the update. The session listener is attached ONCE and + * outlives every turn, so this must never throw: the sandbox-agent registries are plain Sets and a + * thrown handler would corrupt the event stream, so any error is swallowed and logged. + * + * Steps: let the ENOTCONN watcher observe the raw event, extract the update payload (dropping events + * that carry none), record live tool_call ids for client-tool correlation, then hand the update to + * the active turn — or, between turns when no turn owns it, log and drop it. + */ +export function routeSessionEventToActiveTurn( + environment: SessionEnvironment, + remountLocalCwdAfterRuntimeEnotconn: (event: unknown) => void, + event: SessionEvent, +): void { + const { logger, plan } = environment; + try { + remountLocalCwdAfterRuntimeEnotconn(event); + const payload = + event && typeof event === "object" && "payload" in event + ? (event.payload as Record | undefined) + : undefined; + const params = + payload?.params && typeof payload.params === "object" + ? (payload.params as Record) + : undefined; + const update = params?.update ?? payload?.update; + if (!update) return; + // Record live ACP tool_call ids so a paused client_tool can correlate to Claude's bubble + // (session-scoped; a lookup CONSUMES its matched id). + environment.toolCallIndex.record(update); + const turn = environment.currentTurn; + if (turn) { + turn.handleUpdate(update); + } else { + // Between turns (parked/idle): no turn owns this event. Log and drop by decision. + logger(`[keepalive] between-turns event dropped`); + } + } catch (err) { + logger(`session onEvent handler error: ${conciseError(err, plan.harness)}`); + } +} + +/** + * Route one permission gate into the active turn's approval handler. + * + * Data flow: the harness raises a permission request -> the active turn (`environment.currentTurn`) + * decides it. Like the event listener this is attached ONCE and must never throw (a thrown handler + * would corrupt the sandbox-agent registries), so errors are swallowed and logged. + * + * Between turns no turn owns the gate. An approval park is always recorded DURING the active turn + * (the gate fires while a prompt runs, routing through currentTurn), and a parked-on-approval + * session leaves its harness suspended on that gate, so nothing new fires while parked. A gate that + * reaches here is therefore a genuine stray (e.g. a late teardown artifact): reject it by policy so + * it cannot hang. + */ +export function routePermissionRequestToActiveTurn( + environment: SessionEnvironment, + req: SessionPermissionRequest, +): void { + const { logger, plan } = environment; + try { + const turn = environment.currentTurn; + if (turn?.onPermissionRequest) { + turn.onPermissionRequest(req); + return; + } + logger( + `[keepalive] between-turns permission request, cancelling by policy id=${req?.id}`, + ); + void Promise.resolve( + environment.session?.respondPermission?.(req?.id, "reject"), + ).catch(() => {}); + } catch (err) { + logger( + `session onPermissionRequest handler error: ${conciseError(err, plan.harness)}`, + ); + } +} + + diff --git a/services/runner/src/engines/sandbox_agent/session-identity.ts b/services/runner/src/engines/sandbox_agent/session-identity.ts new file mode 100644 index 0000000000..f31c8b897b --- /dev/null +++ b/services/runner/src/engines/sandbox_agent/session-identity.ts @@ -0,0 +1,425 @@ +/** + * Session keep-alive identity and continuation policy. + * + * Pure derivation only: provider config, canonical fingerprints, credential epochs, + * conversation matching, and safe project-scoped pool keys. Mutable lifecycle storage + * belongs in `session-pool.ts`. + */ +import { createHash } from "node:crypto"; + +import { + type AgentRunRequest, + type ChatMessage, + type ContentBlock, + messageText, +} from "../../protocol.ts"; +import { approvalDecisionOf } from "../../responder.ts"; +import type { TeardownReason } from "./teardown.ts"; + +function log(message: string): void { + process.stderr.write(`[keepalive] ${message}\n`); +} + +// --- Config (read once, in one place; mirrors server.ts's env reads) --------- // + +export interface KeepaliveConfig { + enabled: boolean; + ttlMs: number; + approvalTtlMs: number; + poolMax: number; +} + +export type KeepaliveProviderName = "local" | "daytona"; + +const KEEPALIVE_ENV = "AGENTA_RUNNER_SESSION_KEEPALIVE"; +const TTL_ENV = "AGENTA_RUNNER_SESSION_TTL_MS"; +const APPROVAL_TTL_ENV = "AGENTA_RUNNER_SESSION_APPROVAL_TTL_MS"; +const POOL_MAX_ENV = "AGENTA_RUNNER_SESSION_POOL_MAX"; + +const DEFAULT_TTL_MS = 60_000; +const DEFAULT_APPROVAL_TTL_MS = 300_000; +const DEFAULT_POOL_MAX = 8; +const DAYTONA_TTL_ENV = "AGENTA_RUNNER_DAYTONA_SESSION_IDLE_TTL_MS"; +const DAYTONA_POOL_MAX_ENV = "AGENTA_RUNNER_DAYTONA_SESSION_MAX_WARM"; +// Two minutes: the shipping default decided in the plan (about half a cent per parked turn), +// enabled after the E3 live verification. 0 disables keeping Daytona sandboxes running. +const DEFAULT_DAYTONA_TTL_MS = 120_000; +const DEFAULT_DAYTONA_POOL_MAX = 20; + +function positiveIntEnv(name: string, fallback: number): number { + const raw = process.env[name]; + const parsed = raw ? Number(raw) : NaN; + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback; +} + +/** + * Like `positiveIntEnv` but zero is a VALID value, not a fallback trigger. The Daytona idle + * TTL uses this because 0 is its documented off switch; with a nonzero shipping default, a + * positive-only parse would silently turn "0" back into the default. + */ +function nonNegativeIntEnv(name: string, fallback: number): number { + const raw = process.env[name]; + if (raw === undefined || raw.trim() === "") return fallback; + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : fallback; +} + +/** The runner treats only a few explicit truthy spellings as on; default OFF. */ +function boolEnv(name: string): boolean { + const raw = (process.env[name] ?? "").trim().toLowerCase(); + return raw === "1" || raw === "true" || raw === "yes" || raw === "on"; +} + +/** Read one provider's keep-alive config from the environment. */ +export function readKeepaliveConfig( + provider: KeepaliveProviderName, +): KeepaliveConfig { + if (provider === "daytona") { + const ttlMs = nonNegativeIntEnv(DAYTONA_TTL_ENV, DEFAULT_DAYTONA_TTL_MS); + // Keep this live window comfortably below the signed mount-credential lifetime. The + // existing credential-epoch check evicts to cold when those credentials expire. + return { + enabled: ttlMs > 0, + ttlMs, + // Pending approvals on Daytona take the cold path until the F-018 gate plan lands; the + // pool never sees an awaiting_approval park for Daytona today because parkedApproval is + // only set by ACP gates. + approvalTtlMs: ttlMs, + // This budgets billed compute (idle warm sandboxes), deliberately separate from the local + // pool's host-memory budget; Slice 4 adds the strict warm-slot accounting semantics. + poolMax: positiveIntEnv(DAYTONA_POOL_MAX_ENV, DEFAULT_DAYTONA_POOL_MAX), + }; + } + return { + enabled: boolEnv(KEEPALIVE_ENV), + ttlMs: positiveIntEnv(TTL_ENV, DEFAULT_TTL_MS), + approvalTtlMs: positiveIntEnv(APPROVAL_TTL_ENV, DEFAULT_APPROVAL_TTL_MS), + poolMax: positiveIntEnv(POOL_MAX_ENV, DEFAULT_POOL_MAX), + }; +} + +/** + * `poolMax` (and the LRU/TTL eviction it drives) is a LOCAL-provider parameter — "how many + * ~300 MB hot Claude trees fit on this runner host" — never a global one. Mirrors `run-plan.ts`'s + * own sandbox-id resolution (`request.sandbox || SANDBOX_AGENT_PROVIDER env || "local"`), kept + * here (not imported from there) so this module stays engine-agnostic. The pool dispatch + * (`server.ts` `isLocalSandbox`) and the continuity module's own local/remote framing both + * resolve through this one function, so the "local-only" invariant has a single source of truth. + */ +export function resolvesToLocalProvider( + requestSandbox: string | undefined, + env: NodeJS.ProcessEnv = process.env, +): boolean { + return (requestSandbox || env.SANDBOX_AGENT_PROVIDER || "local") === "local"; +} + +// --- Fingerprints and the pool key ------------------------------------------ // + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +/** Deterministic JSON: object keys sorted recursively so equal values hash equal. */ +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) { + return `[${value.map(canonicalJson).join(",")}]`; + } + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries + .map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`) + .join(",")}}`; +} + +/** + * A canonical hash over the config-bearing request fields (the continuation-versus-cold + * decision). Per-turn volatiles are excluded: `messages`, `turnId`, trace propagation + * (`context`), the rotating telemetry headers, and secret VALUES (`secrets` — the credential + * epoch covers rotation, and values must never enter any hash used for logging). The + * tool-callback ENDPOINT is included (routing config); its authorization is a credential and + * lives in the credential epoch instead. + */ +export function configFingerprint(request: AgentRunRequest): string { + const workflow = request.runContext?.workflow; + const shape = { + harness: request.harness ?? null, + sandbox: request.sandbox ?? null, + model: request.model ?? null, + provider: request.provider ?? null, + connection: request.connection ?? null, + deployment: request.deployment ?? null, + endpoint: request.endpoint ?? null, + credentialMode: request.credentialMode ?? null, + agentsMd: request.agentsMd ?? null, + systemPrompt: request.systemPrompt ?? null, + appendSystemPrompt: request.appendSystemPrompt ?? null, + tools: request.tools ?? null, + skills: request.skills ?? null, + customTools: request.customTools ?? null, + mcpServers: request.mcpServers ?? null, + toolCallbackEndpoint: request.toolCallback?.endpoint ?? null, + permissions: request.permissions ?? null, + sandboxPermission: request.sandboxPermission ?? null, + harnessFiles: request.harnessFiles ?? null, + workflowRevision: workflow?.revision + ? { + id: workflow.revision.id ?? null, + version: workflow.revision.version ?? null, + } + : null, + isDraft: workflow?.is_draft ?? null, + }; + return sha256(canonicalJson(shape)); +} + +function collectToolCallIds( + content: string | ContentBlock[] | undefined, + into: string[], + seen: Set, +): void { + if (!Array.isArray(content)) return; + for (const block of content) { + if (!block) continue; + if ( + (block.type === "tool_call" || block.type === "tool_result") && + typeof block.toolCallId === "string" && + block.toolCallId && + !seen.has(block.toolCallId) + ) { + seen.add(block.toolCallId); + into.push(block.toolCallId); + } + } +} + +/** + * A hash over the conversation the server received (the FE's pruned array): the ordered user + * message texts, the ordered tool-call ids across every message, and the user-message count. + * Assistant TEXT is deliberately ignored, so a live session that has already answered a plain + * user turn matches the next request's prefix (the FE's assistant turn contributes nothing). + * Tool-call ids ARE included, so an edited history trips a mismatch and degrades to cold + * replay rather than continuing wrongly. Ids are DEDUPED (unique, first-seen order): a resolved + * tool call rides the wire as a `tool_call` block PLUS a `tool_result` block sharing one id + * (vercel `messages.py` `_tool_part_blocks`), and the park-time prediction + * (`expectedNextHistoryFingerprint`) folds each emitted id in once — dedupe makes the two agree + * while a genuinely different id SET still mismatches. + * + * The dispatch stores the fingerprint the next request is EXPECTED to hash to (see + * `expectedNextHistoryFingerprint`), and checks the next request against the fingerprint of its + * PRIOR messages (everything before the new user tail), so a plain conversational continuation + * matches and any divergence falls to cold. + */ +export function historyFingerprint(messages: readonly ChatMessage[]): string { + const userTexts: string[] = []; + const toolCallIds: string[] = []; + const seenIds = new Set(); + let promptCount = 0; + for (const message of messages) { + if (message.role === "user") { + promptCount += 1; + userTexts.push(messageText(message.content)); + } + collectToolCallIds(message.content, toolCallIds, seenIds); + } + return sha256(canonicalJson({ userTexts, toolCallIds, promptCount })); +} + +/** + * The fingerprint a park should record so the NEXT request's prior conversation matches it: + * the full messages this turn ran, plus the tool-call ids the turn itself emitted, folded in + * as one synthetic trailing assistant message. + * + * Why: the FE keeps an assistant turn iff it has an answer part (`agentRequest.ts` + * `isAnswerPart`: non-empty text, `tool-*`/`dynamic-tool`, or file). So a tool-calling turn's + * ids ALWAYS appear in the next request's prior messages, and a fully empty assistant turn is + * pruned but contributes neither text nor ids — the prediction is deterministic either way + * (assistant text is not hashed). An id divergence still trips a mismatch and falls to cold. + */ +export function expectedNextHistoryFingerprint( + messages: readonly ChatMessage[], + emittedToolCallIds: readonly string[], +): string { + if (emittedToolCallIds.length === 0) return historyFingerprint(messages); + const syntheticAssistantTurn: ChatMessage = { + role: "assistant", + content: emittedToolCallIds.map((id) => ({ + type: "tool_call", + toolCallId: id, + })), + }; + return historyFingerprint([...messages, syntheticAssistantTurn]); +} + +/** + * The prior conversation for a continuation check: everything before the request's new user + * tail. Mirrors `transcript.priorMessages` for the trailing-user case (the playground always + * sends the new turn as the last user message), without importing that do-not-touch module. + */ +export function priorConversation(request: AgentRunRequest): ChatMessage[] { + const messages = request.messages ?? []; + if (messages.length && messages[messages.length - 1].role === "user") { + return messages.slice(0, -1); + } + return messages.slice(); +} + +/** + * The approval decision (allow/deny) the incoming request carries for a specific parked gate's + * tool-call id, or undefined when the request has no approval envelope for that id. Reuses the + * cold path's `approvalDecisionOf` (responder.ts) to parse the `{approved}` envelope, and matches + * strictly by toolCallId (the parked gate's id) — never by name+args — so a live resume answers + * exactly the gate that parked. An incoming reply for a different id, or a plain user message, + * yields undefined and the dispatch degrades to cold. + */ +export function approvalDecisionForToolCall( + request: AgentRunRequest, + toolCallId: string, +): "allow" | "deny" | undefined { + if (!toolCallId) return undefined; + for (const message of request.messages ?? []) { + const content = message?.content; + if (!Array.isArray(content)) continue; + for (const block of content) { + if (block?.type !== "tool_result" || block.toolCallId !== toolCallId) { + continue; + } + const decision = approvalDecisionOf(block); + if (decision !== undefined) return decision; + } + } + return undefined; +} + +/** + * True when the request's tail is a fresh user message with text and NOT an approval envelope. + * A continuation only takes the live path for a plain new user turn; an approval reply (a + * trailing tool-role message, or a user turn carrying a tool_result) stays cold here. + */ +export function tailIsFreshUserMessage(request: AgentRunRequest): boolean { + const messages = request.messages ?? []; + const tail = messages[messages.length - 1]; + if (!tail || tail.role !== "user") return false; + if (!messageText(tail.content).trim()) return false; + if (Array.isArray(tail.content)) { + const carriesToolTurn = tail.content.some( + (block) => block?.type === "tool_result" || block?.type === "tool_call", + ); + if (carriesToolTurn) return false; + } + return true; +} + +/** + * The credential epoch bounds how long a parked session may reuse its baked credentials. It is + * a PROCESS-LOCAL hash over the actual resolved secret VALUES plus the tool-callback auth (held + * only in runner memory — never logged, persisted, or emitted), combined with the mount + * credential expiry. A rotated same-slug secret changes the hash; an elapsed expiry invalidates + * the epoch. Either way the dispatch evicts and cold-starts with fresh credentials. + */ +export interface CredentialEpoch { + /** sha256 over canonical(secrets) + tool-callback auth. In-memory only; never surfaced. */ + secretsHash: string; + /** Mount credential expiry as epoch millis, or undefined when the sign response had none. */ + mountExpiresAtMs?: number; +} + +export function computeCredentialEpoch( + request: AgentRunRequest, + mountExpiresAt?: string, +): CredentialEpoch { + const material = canonicalJson({ + secrets: request.secrets ?? {}, + toolCallbackAuth: request.toolCallback?.authorization ?? null, + }); + const parsed = mountExpiresAt ? Date.parse(mountExpiresAt) : NaN; + return { + secretsHash: sha256(material), + mountExpiresAtMs: Number.isFinite(parsed) ? parsed : undefined, + }; +} + +/** + * Whether a parked session's MOUNT credentials have already expired, ignoring the secret material + * hash entirely. This answers only "can the parked environment still write its durable cwd?". + * + * The approval-resume path uses this instead of `credentialEpochValid`: a resume must NOT require + * the resume request's re-minted credentials to MATCH the parked ones (a fresh /run mints fresh + * short-lived material every time, so they practically never match), but an expired mount means + * the parked cwd can no longer be written, so it must still evict to cold. + */ +export function mountCredentialsExpired( + epoch: CredentialEpoch, + now = Date.now(), +): boolean { + return epoch.mountExpiresAtMs !== undefined && now >= epoch.mountExpiresAtMs; +} + +/** + * Why a parked epoch is no longer usable for an incoming request's epoch, or undefined when it + * still is. The two failure modes are distinguished so diagnosis works from logs: + * - `credentials-expired` — the mount credential's lifetime elapsed (time bound). + * - `credentials-rotated` — the resolved secret/tool-auth material changed (a rotated same-slug + * secret, a different tool-callback bearer). + */ +export function credentialEpochMismatch( + parked: CredentialEpoch, + incoming: CredentialEpoch, + now = Date.now(), +): "credentials-expired" | "credentials-rotated" | undefined { + if (mountCredentialsExpired(parked, now)) return "credentials-expired"; + if (parked.secretsHash !== incoming.secretsHash) return "credentials-rotated"; + return undefined; +} + +/** + * Whether a parked epoch is still valid for an incoming request's epoch. Invalid (evict, cold) + * when the mount credential expired, or the resolved secret/tool-auth material changed. Thin + * wrapper over `credentialEpochMismatch` for callers that only need the boolean. + */ +export function credentialEpochValid( + parked: CredentialEpoch, + incoming: CredentialEpoch, + now = Date.now(), +): boolean { + return credentialEpochMismatch(parked, incoming, now) === undefined; +} + +/** Which project-scope source produced a pool key: the service-stamped run context, or the mount. */ +export type PoolScopeSource = "run-context" | "mount"; + +/** A pool key plus the scope source that produced it (for the greppable `[keepalive] scope=` log). */ +export interface PoolScope { + key: string; + source: PoolScopeSource; +} + +/** + * The pool key: `:`. The project scope is PREFERRED from the run context the + * service stamps server-side (`runContext.project.id`), and FALLS BACK to the mount's owning + * project id when the run context carries none. Provider separation does not need another key + * segment: providers have separate pools, and `configFingerprint` includes `request.sandbox`. + * The run-context id is the trustworthy source: the + * service derives it from its own request state (never from a caller-supplied wire field), so it + * does not depend on a durable mount existing. The mount scope stays as the fallback for the + * transition and for runs without a stamped project. + * + * Returns null when there is no session id, or when NEITHER source yields a project scope — such a + * request MUST NOT park (there is no safe key that separates callers), and the dispatch runs it + * fully cold. This no-scope-no-park rule is the keep-alive safety invariant and is unchanged. + */ +export function poolKeyFor( + request: AgentRunRequest, + mountProjectId: string | undefined, +): PoolScope | null { + const sessionId = request.sessionId?.trim(); + if (!sessionId) return null; + const runContextProject = request.runContext?.project?.id?.trim(); + if (runContextProject) { + return { key: `${runContextProject}:${sessionId}`, source: "run-context" }; + } + const mount = mountProjectId?.trim(); + if (mount) return { key: `${mount}:${sessionId}`, source: "mount" }; + return null; +} diff --git a/services/runner/src/engines/sandbox_agent/session-pool.ts b/services/runner/src/engines/sandbox_agent/session-pool.ts index 1cde9714d2..c0bd410c5b 100644 --- a/services/runner/src/engines/sandbox_agent/session-pool.ts +++ b/services/runner/src/engines/sandbox_agent/session-pool.ts @@ -1,439 +1,17 @@ /** - * Session keep-alive pool: normal turn boundaries, local sandbox, flag-gated off by default. + * Mutable lifecycle storage for parked sandbox-agent sessions. * - * Background: today the runner destroys the sandbox and harness session at the end of every - * turn (`sandbox_agent.ts` teardown). Keep-alive parks a live session for a short TTL so the - * next message in the same conversation continues the same live harness process, keeping its - * native memory. If the window expires or anything mismatches, the dispatch falls back to - * today's cold path. Nothing gets worse than today; with the flag off nothing here runs. - * - * This module is engine-agnostic: it holds opaque `environment` handles plus the metadata the - * dispatch needs to decide continue-versus-cold (two fingerprints, a credential epoch, an LRU - * timestamp, a state) and a complete idempotent `teardown(reason)` closure the engine supplies. It - * never imports the engine, so it stays a pure map + timer + policy unit. + * Identity, fingerprint, credential, provider, and key policy lives in + * `session-identity.ts`; this module owns only the map, timers, capacity, state + * transitions, and idempotent teardown. */ -import { createHash } from "node:crypto"; - -import { - type AgentRunRequest, - type ChatMessage, - type ContentBlock, - messageText, -} from "../../protocol.ts"; -import { approvalDecisionOf } from "../../responder.ts"; +import type { CredentialEpoch, KeepaliveConfig } from "./session-identity.ts"; import type { TeardownReason } from "./teardown.ts"; function log(message: string): void { process.stderr.write(`[keepalive] ${message}\n`); } -// --- Config (read once, in one place; mirrors server.ts's env reads) --------- // - -export interface KeepaliveConfig { - enabled: boolean; - ttlMs: number; - approvalTtlMs: number; - poolMax: number; -} - -export type KeepaliveProviderName = "local" | "daytona"; - -const KEEPALIVE_ENV = "AGENTA_RUNNER_SESSION_KEEPALIVE"; -const TTL_ENV = "AGENTA_RUNNER_SESSION_TTL_MS"; -const APPROVAL_TTL_ENV = "AGENTA_RUNNER_SESSION_APPROVAL_TTL_MS"; -const POOL_MAX_ENV = "AGENTA_RUNNER_SESSION_POOL_MAX"; - -const DEFAULT_TTL_MS = 60_000; -const DEFAULT_APPROVAL_TTL_MS = 300_000; -const DEFAULT_POOL_MAX = 8; -const DAYTONA_TTL_ENV = "AGENTA_RUNNER_DAYTONA_SESSION_IDLE_TTL_MS"; -const DAYTONA_POOL_MAX_ENV = "AGENTA_RUNNER_DAYTONA_SESSION_MAX_WARM"; -// Two minutes: the shipping default decided in the plan (about half a cent per parked turn), -// enabled after the E3 live verification. 0 disables keeping Daytona sandboxes running. -const DEFAULT_DAYTONA_TTL_MS = 120_000; -const DEFAULT_DAYTONA_POOL_MAX = 20; - -function positiveIntEnv(name: string, fallback: number): number { - const raw = process.env[name]; - const parsed = raw ? Number(raw) : NaN; - return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback; -} - -/** - * Like `positiveIntEnv` but zero is a VALID value, not a fallback trigger. The Daytona idle - * TTL uses this because 0 is its documented off switch; with a nonzero shipping default, a - * positive-only parse would silently turn "0" back into the default. - */ -function nonNegativeIntEnv(name: string, fallback: number): number { - const raw = process.env[name]; - if (raw === undefined || raw.trim() === "") return fallback; - const parsed = Number(raw); - return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : fallback; -} - -/** The runner treats only a few explicit truthy spellings as on; default OFF. */ -function boolEnv(name: string): boolean { - const raw = (process.env[name] ?? "").trim().toLowerCase(); - return raw === "1" || raw === "true" || raw === "yes" || raw === "on"; -} - -/** Read one provider's keep-alive config from the environment. */ -export function readKeepaliveConfig( - provider: KeepaliveProviderName, -): KeepaliveConfig { - if (provider === "daytona") { - const ttlMs = nonNegativeIntEnv(DAYTONA_TTL_ENV, DEFAULT_DAYTONA_TTL_MS); - // Keep this live window comfortably below the signed mount-credential lifetime. The - // existing credential-epoch check evicts to cold when those credentials expire. - return { - enabled: ttlMs > 0, - ttlMs, - // Pending approvals on Daytona take the cold path until the F-018 gate plan lands; the - // pool never sees an awaiting_approval park for Daytona today because parkedApproval is - // only set by ACP gates. - approvalTtlMs: ttlMs, - // This budgets billed compute (idle warm sandboxes), deliberately separate from the local - // pool's host-memory budget; Slice 4 adds the strict warm-slot accounting semantics. - poolMax: positiveIntEnv( - DAYTONA_POOL_MAX_ENV, - DEFAULT_DAYTONA_POOL_MAX, - ), - }; - } - return { - enabled: boolEnv(KEEPALIVE_ENV), - ttlMs: positiveIntEnv(TTL_ENV, DEFAULT_TTL_MS), - approvalTtlMs: positiveIntEnv(APPROVAL_TTL_ENV, DEFAULT_APPROVAL_TTL_MS), - poolMax: positiveIntEnv(POOL_MAX_ENV, DEFAULT_POOL_MAX), - }; -} - -/** - * `poolMax` (and the LRU/TTL eviction it drives) is a LOCAL-provider parameter — "how many - * ~300 MB hot Claude trees fit on this runner host" — never a global one. Mirrors `run-plan.ts`'s - * own sandbox-id resolution (`request.sandbox || SANDBOX_AGENT_PROVIDER env || "local"`), kept - * here (not imported from there) so this module stays engine-agnostic. The pool dispatch - * (`server.ts` `isLocalSandbox`) and the continuity module's own local/remote framing both - * resolve through this one function, so the "local-only" invariant has a single source of truth. - */ -export function resolvesToLocalProvider( - requestSandbox: string | undefined, - env: NodeJS.ProcessEnv = process.env, -): boolean { - return (requestSandbox || env.SANDBOX_AGENT_PROVIDER || "local") === "local"; -} - -// --- Fingerprints and the pool key ------------------------------------------ // - -function sha256(value: string): string { - return createHash("sha256").update(value).digest("hex"); -} - -/** Deterministic JSON: object keys sorted recursively so equal values hash equal. */ -function canonicalJson(value: unknown): string { - if (value === null || typeof value !== "object") return JSON.stringify(value); - if (Array.isArray(value)) { - return `[${value.map(canonicalJson).join(",")}]`; - } - const entries = Object.entries(value as Record) - .filter(([, v]) => v !== undefined) - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); - return `{${entries - .map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`) - .join(",")}}`; -} - -/** - * A canonical hash over the config-bearing request fields (the continuation-versus-cold - * decision). Per-turn volatiles are excluded: `messages`, `turnId`, trace propagation - * (`context`), the rotating telemetry headers, and secret VALUES (`secrets` — the credential - * epoch covers rotation, and values must never enter any hash used for logging). The - * tool-callback ENDPOINT is included (routing config); its authorization is a credential and - * lives in the credential epoch instead. - */ -export function configFingerprint(request: AgentRunRequest): string { - const workflow = request.runContext?.workflow; - const shape = { - harness: request.harness ?? null, - sandbox: request.sandbox ?? null, - model: request.model ?? null, - provider: request.provider ?? null, - connection: request.connection ?? null, - deployment: request.deployment ?? null, - endpoint: request.endpoint ?? null, - credentialMode: request.credentialMode ?? null, - agentsMd: request.agentsMd ?? null, - systemPrompt: request.systemPrompt ?? null, - appendSystemPrompt: request.appendSystemPrompt ?? null, - tools: request.tools ?? null, - skills: request.skills ?? null, - customTools: request.customTools ?? null, - mcpServers: request.mcpServers ?? null, - toolCallbackEndpoint: request.toolCallback?.endpoint ?? null, - permissions: request.permissions ?? null, - sandboxPermission: request.sandboxPermission ?? null, - harnessFiles: request.harnessFiles ?? null, - workflowRevision: workflow?.revision - ? { - id: workflow.revision.id ?? null, - version: workflow.revision.version ?? null, - } - : null, - isDraft: workflow?.is_draft ?? null, - }; - return sha256(canonicalJson(shape)); -} - -function collectToolCallIds( - content: string | ContentBlock[] | undefined, - into: string[], - seen: Set, -): void { - if (!Array.isArray(content)) return; - for (const block of content) { - if (!block) continue; - if ( - (block.type === "tool_call" || block.type === "tool_result") && - typeof block.toolCallId === "string" && - block.toolCallId && - !seen.has(block.toolCallId) - ) { - seen.add(block.toolCallId); - into.push(block.toolCallId); - } - } -} - -/** - * A hash over the conversation the server received (the FE's pruned array): the ordered user - * message texts, the ordered tool-call ids across every message, and the user-message count. - * Assistant TEXT is deliberately ignored, so a live session that has already answered a plain - * user turn matches the next request's prefix (the FE's assistant turn contributes nothing). - * Tool-call ids ARE included, so an edited history trips a mismatch and degrades to cold - * replay rather than continuing wrongly. Ids are DEDUPED (unique, first-seen order): a resolved - * tool call rides the wire as a `tool_call` block PLUS a `tool_result` block sharing one id - * (vercel `messages.py` `_tool_part_blocks`), and the park-time prediction - * (`expectedNextHistoryFingerprint`) folds each emitted id in once — dedupe makes the two agree - * while a genuinely different id SET still mismatches. - * - * The dispatch stores the fingerprint the next request is EXPECTED to hash to (see - * `expectedNextHistoryFingerprint`), and checks the next request against the fingerprint of its - * PRIOR messages (everything before the new user tail), so a plain conversational continuation - * matches and any divergence falls to cold. - */ -export function historyFingerprint(messages: readonly ChatMessage[]): string { - const userTexts: string[] = []; - const toolCallIds: string[] = []; - const seenIds = new Set(); - let promptCount = 0; - for (const message of messages) { - if (message.role === "user") { - promptCount += 1; - userTexts.push(messageText(message.content)); - } - collectToolCallIds(message.content, toolCallIds, seenIds); - } - return sha256(canonicalJson({ userTexts, toolCallIds, promptCount })); -} - -/** - * The fingerprint a park should record so the NEXT request's prior conversation matches it: - * the full messages this turn ran, plus the tool-call ids the turn itself emitted, folded in - * as one synthetic trailing assistant message. - * - * Why: the FE keeps an assistant turn iff it has an answer part (`agentRequest.ts` - * `isAnswerPart`: non-empty text, `tool-*`/`dynamic-tool`, or file). So a tool-calling turn's - * ids ALWAYS appear in the next request's prior messages, and a fully empty assistant turn is - * pruned but contributes neither text nor ids — the prediction is deterministic either way - * (assistant text is not hashed). An id divergence still trips a mismatch and falls to cold. - */ -export function expectedNextHistoryFingerprint( - messages: readonly ChatMessage[], - emittedToolCallIds: readonly string[], -): string { - if (emittedToolCallIds.length === 0) return historyFingerprint(messages); - const syntheticAssistantTurn: ChatMessage = { - role: "assistant", - content: emittedToolCallIds.map((id) => ({ - type: "tool_call", - toolCallId: id, - })), - }; - return historyFingerprint([...messages, syntheticAssistantTurn]); -} - -/** - * The prior conversation for a continuation check: everything before the request's new user - * tail. Mirrors `transcript.priorMessages` for the trailing-user case (the playground always - * sends the new turn as the last user message), without importing that do-not-touch module. - */ -export function priorConversation(request: AgentRunRequest): ChatMessage[] { - const messages = request.messages ?? []; - if (messages.length && messages[messages.length - 1].role === "user") { - return messages.slice(0, -1); - } - return messages.slice(); -} - -/** - * The approval decision (allow/deny) the incoming request carries for a specific parked gate's - * tool-call id, or undefined when the request has no approval envelope for that id. Reuses the - * cold path's `approvalDecisionOf` (responder.ts) to parse the `{approved}` envelope, and matches - * strictly by toolCallId (the parked gate's id) — never by name+args — so a live resume answers - * exactly the gate that parked. An incoming reply for a different id, or a plain user message, - * yields undefined and the dispatch degrades to cold. - */ -export function approvalDecisionForToolCall( - request: AgentRunRequest, - toolCallId: string, -): "allow" | "deny" | undefined { - if (!toolCallId) return undefined; - for (const message of request.messages ?? []) { - const content = message?.content; - if (!Array.isArray(content)) continue; - for (const block of content) { - if (block?.type !== "tool_result" || block.toolCallId !== toolCallId) { - continue; - } - const decision = approvalDecisionOf(block); - if (decision !== undefined) return decision; - } - } - return undefined; -} - -/** - * True when the request's tail is a fresh user message with text and NOT an approval envelope. - * A continuation only takes the live path for a plain new user turn; an approval reply (a - * trailing tool-role message, or a user turn carrying a tool_result) stays cold here. - */ -export function tailIsFreshUserMessage(request: AgentRunRequest): boolean { - const messages = request.messages ?? []; - const tail = messages[messages.length - 1]; - if (!tail || tail.role !== "user") return false; - if (!messageText(tail.content).trim()) return false; - if (Array.isArray(tail.content)) { - const carriesToolTurn = tail.content.some( - (block) => block?.type === "tool_result" || block?.type === "tool_call", - ); - if (carriesToolTurn) return false; - } - return true; -} - -/** - * The credential epoch bounds how long a parked session may reuse its baked credentials. It is - * a PROCESS-LOCAL hash over the actual resolved secret VALUES plus the tool-callback auth (held - * only in runner memory — never logged, persisted, or emitted), combined with the mount - * credential expiry. A rotated same-slug secret changes the hash; an elapsed expiry invalidates - * the epoch. Either way the dispatch evicts and cold-starts with fresh credentials. - */ -export interface CredentialEpoch { - /** sha256 over canonical(secrets) + tool-callback auth. In-memory only; never surfaced. */ - secretsHash: string; - /** Mount credential expiry as epoch millis, or undefined when the sign response had none. */ - mountExpiresAtMs?: number; -} - -export function computeCredentialEpoch( - request: AgentRunRequest, - mountExpiresAt?: string, -): CredentialEpoch { - const material = canonicalJson({ - secrets: request.secrets ?? {}, - toolCallbackAuth: request.toolCallback?.authorization ?? null, - }); - const parsed = mountExpiresAt ? Date.parse(mountExpiresAt) : NaN; - return { - secretsHash: sha256(material), - mountExpiresAtMs: Number.isFinite(parsed) ? parsed : undefined, - }; -} - -/** - * Whether a parked session's MOUNT credentials have already expired, ignoring the secret material - * hash entirely. This answers only "can the parked environment still write its durable cwd?". - * - * The approval-resume path uses this instead of `credentialEpochValid`: a resume must NOT require - * the resume request's re-minted credentials to MATCH the parked ones (a fresh /run mints fresh - * short-lived material every time, so they practically never match), but an expired mount means - * the parked cwd can no longer be written, so it must still evict to cold. - */ -export function mountCredentialsExpired( - epoch: CredentialEpoch, - now = Date.now(), -): boolean { - return epoch.mountExpiresAtMs !== undefined && now >= epoch.mountExpiresAtMs; -} - -/** - * Why a parked epoch is no longer usable for an incoming request's epoch, or undefined when it - * still is. The two failure modes are distinguished so diagnosis works from logs: - * - `credentials-expired` — the mount credential's lifetime elapsed (time bound). - * - `credentials-rotated` — the resolved secret/tool-auth material changed (a rotated same-slug - * secret, a different tool-callback bearer). - */ -export function credentialEpochMismatch( - parked: CredentialEpoch, - incoming: CredentialEpoch, - now = Date.now(), -): "credentials-expired" | "credentials-rotated" | undefined { - if (mountCredentialsExpired(parked, now)) return "credentials-expired"; - if (parked.secretsHash !== incoming.secretsHash) return "credentials-rotated"; - return undefined; -} - -/** - * Whether a parked epoch is still valid for an incoming request's epoch. Invalid (evict, cold) - * when the mount credential expired, or the resolved secret/tool-auth material changed. Thin - * wrapper over `credentialEpochMismatch` for callers that only need the boolean. - */ -export function credentialEpochValid( - parked: CredentialEpoch, - incoming: CredentialEpoch, - now = Date.now(), -): boolean { - return credentialEpochMismatch(parked, incoming, now) === undefined; -} - -/** Which project-scope source produced a pool key: the service-stamped run context, or the mount. */ -export type PoolScopeSource = "run-context" | "mount"; - -/** A pool key plus the scope source that produced it (for the greppable `[keepalive] scope=` log). */ -export interface PoolScope { - key: string; - source: PoolScopeSource; -} - -/** - * The pool key: `:`. The project scope is PREFERRED from the run context the - * service stamps server-side (`runContext.project.id`), and FALLS BACK to the mount's owning - * project id when the run context carries none. Provider separation does not need another key - * segment: providers have separate pools, and `configFingerprint` includes `request.sandbox`. - * The run-context id is the trustworthy source: the - * service derives it from its own request state (never from a caller-supplied wire field), so it - * does not depend on a durable mount existing. The mount scope stays as the fallback for the - * transition and for runs without a stamped project. - * - * Returns null when there is no session id, or when NEITHER source yields a project scope — such a - * request MUST NOT park (there is no safe key that separates callers), and the dispatch runs it - * fully cold. This no-scope-no-park rule is the keep-alive safety invariant and is unchanged. - */ -export function poolKeyFor( - request: AgentRunRequest, - mountProjectId: string | undefined, -): PoolScope | null { - const sessionId = request.sessionId?.trim(); - if (!sessionId) return null; - const runContextProject = request.runContext?.project?.id?.trim(); - if (runContextProject) { - return { key: `${runContextProject}:${sessionId}`, source: "run-context" }; - } - const mount = mountProjectId?.trim(); - if (mount) return { key: `${mount}:${sessionId}`, source: "mount" }; - return null; -} - // --- The pool --------------------------------------------------------------- // export type SessionState = "busy" | "idle" | "awaiting_approval" | "destroyed"; diff --git a/services/runner/src/engines/sandbox_agent/tool-mcp-assets.ts b/services/runner/src/engines/sandbox_agent/tool-mcp-assets.ts index 59ddd86a06..aca0e14fd5 100644 --- a/services/runner/src/engines/sandbox_agent/tool-mcp-assets.ts +++ b/services/runner/src/engines/sandbox_agent/tool-mcp-assets.ts @@ -25,6 +25,7 @@ import { join } from "node:path"; import type { AdvertisedToolSpec } from "../../tools/public-spec.ts"; import { PKG_ROOT } from "./daemon.ts"; +import type { SandboxFilePort } from "./sandbox-ports.ts"; type Log = (message: string) => void; @@ -73,7 +74,7 @@ export interface ToolMcpAssets { * the specs are the public advertisement shape). */ export async function uploadToolMcpAssets( - sandbox: any, + sandbox: SandboxFilePort, destDir: string, specs: AdvertisedToolSpec[], log: Log = () => {}, diff --git a/services/runner/src/engines/sandbox_agent/usage.ts b/services/runner/src/engines/sandbox_agent/usage.ts index 1d08b3447b..6b743c85c3 100644 --- a/services/runner/src/engines/sandbox_agent/usage.ts +++ b/services/runner/src/engines/sandbox_agent/usage.ts @@ -2,9 +2,34 @@ import { existsSync, readFileSync } from "node:fs"; import type { AgentRunResult, AgentUsage } from "../../protocol.ts"; +interface UsageReader { + readFsFile?: (input: { path: string }) => Promise; +} + +function finiteNonNegative(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? value + : undefined; +} + +/** Parse untrusted extension output without letting malformed JSON cross the runner contract. */ +export function parseRunUsage(value: unknown): AgentUsage | undefined { + if (!value || typeof value !== "object") return undefined; + const usage = value as Record; + const input = finiteNonNegative(usage.input); + const output = finiteNonNegative(usage.output); + const total = finiteNonNegative(usage.total); + const cost = finiteNonNegative(usage.cost); + if (input === undefined || output === undefined || total === undefined) { + return undefined; + } + if (total <= 0 && (cost ?? 0) <= 0) return undefined; + return { input, output, total, cost: cost ?? 0 }; +} + /** Read the run-total usage Pi wrote on agent_end, from local fs or the sandbox FS API. */ export async function readRunUsage( - sandbox: any, + sandbox: UsageReader, path: string | undefined, isDaytona: boolean, ): Promise { @@ -12,14 +37,14 @@ export async function readRunUsage( try { let raw: string; if (isDaytona) { + if (!sandbox.readFsFile) return undefined; const bytes = await sandbox.readFsFile({ path }); raw = typeof bytes === "string" ? bytes : new TextDecoder().decode(bytes); } else { if (!existsSync(path)) return undefined; raw = readFileSync(path, "utf-8"); } - const u = JSON.parse(raw); - return u && u.total > 0 ? u : undefined; + return parseRunUsage(JSON.parse(raw)); } catch { return undefined; } @@ -27,12 +52,21 @@ export async function readRunUsage( /** Combine prompt token counts with stream cost when no Pi usage writeback exists. */ export function mergePromptAndStreamUsage( - promptResult: any, + promptResult: unknown, streamUsage: AgentUsage | undefined, ): AgentUsage | undefined { - const promptUsage = promptResult?.usage; - const inputTokens = promptUsage?.inputTokens ?? streamUsage?.input ?? 0; - const outputTokens = promptUsage?.outputTokens ?? streamUsage?.output ?? 0; + const promptUsage = + promptResult && typeof promptResult === "object" + ? ( + promptResult as { + usage?: { inputTokens?: unknown; outputTokens?: unknown }; + } + ).usage + : undefined; + const inputTokens = + finiteNonNegative(promptUsage?.inputTokens) ?? streamUsage?.input ?? 0; + const outputTokens = + finiteNonNegative(promptUsage?.outputTokens) ?? streamUsage?.output ?? 0; const total = inputTokens + outputTokens || streamUsage?.total || 0; const cost = streamUsage?.cost ?? 0; return total > 0 || cost > 0 @@ -47,10 +81,10 @@ export async function resolveRunUsage({ promptResult, streamUsage, }: { - sandbox: any; + sandbox: UsageReader; usageOutPath: string | undefined; isDaytona: boolean; - promptResult: any; + promptResult: unknown; streamUsage: AgentUsage | undefined; }): Promise { return ( diff --git a/services/runner/src/engines/sandbox_agent/workspace.ts b/services/runner/src/engines/sandbox_agent/workspace.ts index 3a1f8b04fe..c280d665cd 100644 --- a/services/runner/src/engines/sandbox_agent/workspace.ts +++ b/services/runner/src/engines/sandbox_agent/workspace.ts @@ -3,6 +3,11 @@ import { dirname, join } from "node:path"; import type { RunPlan } from "./run-plan.ts"; import { uploadDirToSandbox } from "./pi-assets.ts"; +import type { + SandboxFilePort, + SandboxProcessPort, + SandboxWorkspacePort, +} from "./sandbox-ports.ts"; type Log = (message: string) => void; @@ -10,8 +15,24 @@ export interface Workspace { cleanup: () => Promise; } +type RemoteWorkspacePort = SandboxFilePort & + Partial>; + +function assertRemoteWorkspacePort( + sandbox: SandboxWorkspacePort, +): asserts sandbox is RemoteWorkspacePort { + if ( + typeof sandbox.mkdirFs !== "function" || + typeof sandbox.writeFsFile !== "function" + ) { + throw new Error( + "Daytona workspace requires sandbox filesystem capabilities", + ); + } +} + export interface PrepareWorkspaceInput { - sandbox: any; + sandbox: SandboxWorkspacePort; plan: Pick< RunPlan, | "isDaytona" @@ -54,6 +75,7 @@ export async function prepareWorkspace({ plan.acpAgent === "claude" ? "CLAUDE.md" : "AGENTS.md"; if (plan.isDaytona) { + assertRemoteWorkspacePort(sandbox); await sandbox.mkdirFs({ path: plan.cwd }).catch((err: Error) => { log(`workspace mkdir skipped: ${err.message}`); }); diff --git a/services/runner/src/server.ts b/services/runner/src/server.ts index a6a8d8bb59..4bb123950d 100644 --- a/services/runner/src/server.ts +++ b/services/runner/src/server.ts @@ -53,10 +53,12 @@ import { priorConversation, readKeepaliveConfig, resolvesToLocalProvider, - SessionPool, tailIsFreshUserMessage, type KeepaliveConfig, type KeepaliveProviderName, +} from "./engines/sandbox_agent/session-identity.ts"; +import { + SessionPool, type LiveSession, } from "./engines/sandbox_agent/session-pool.ts"; import { runnerInfo } from "./version.ts"; @@ -575,11 +577,7 @@ export async function runWithKeepalive( klog(`mismatch (${mismatch}) key=${key}; evict + cold`); // Await: the old teardown unmounts the same durable cwd the cold acquire is about to // mount — they must never overlap. - await pool.evict( - key, - `mismatch:${mismatch}`, - "compatibility-mismatch", - ); + await pool.evict(key, `mismatch:${mismatch}`, "compatibility-mismatch"); return coldAndPark(); } @@ -756,11 +754,7 @@ export async function runWithKeepalive( // environment can never be destroyed by this branch). Supersede — destroy the parked one and // cold-start — awaited so its teardown cannot overlap our acquire. klog(`evict (supersede-${existing.state}) key=${key}; cold`); - await pool.evict( - key, - `supersede-${existing.state}`, - "failed-turn", - ); + await pool.evict(key, `supersede-${existing.state}`, "failed-turn"); } else { klog(`miss key=${key}; cold`); } @@ -781,11 +775,9 @@ const keepalivePools: Record< SessionPool > = { local: new SessionPool(keepaliveConfigs.local), - daytona: new SessionPool( - keepaliveConfigs.daytona, - klog, - { strictCapacity: true }, - ), + daytona: new SessionPool(keepaliveConfigs.daytona, klog, { + strictCapacity: true, + }), }; const runAgent: RunAgent = (request, emit, signal, options) => { @@ -1116,11 +1108,7 @@ if (isEntrypoint(import.meta.url)) { onCleanup: async (timeoutMs?: number) => { await Promise.all( Object.values(keepalivePools).map((pool) => - pool.destroyAll( - timeoutMs, - "shutdown-idle", - "shutdown-in-flight", - ), + pool.destroyAll(timeoutMs, "shutdown-idle", "shutdown-in-flight"), ), ); await destroyInFlightSandboxes(timeoutMs, "shutdown-in-flight"); diff --git a/services/runner/tests/unit/sandbox-agent-usage.test.ts b/services/runner/tests/unit/sandbox-agent-usage.test.ts index a43b7c2bb9..70cac78e5d 100644 --- a/services/runner/tests/unit/sandbox-agent-usage.test.ts +++ b/services/runner/tests/unit/sandbox-agent-usage.test.ts @@ -18,7 +18,8 @@ import { const dirs: string[] = []; afterEach(() => { - for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); + for (const dir of dirs.splice(0)) + rmSync(dir, { recursive: true, force: true }); }); describe("readRunUsage", () => { @@ -26,14 +27,37 @@ describe("readRunUsage", () => { const dir = mkdtempSync(join(tmpdir(), "agenta-usage-test-")); dirs.push(dir); const file = join(dir, "usage.json"); - writeFileSync(file, JSON.stringify({ input: 2, output: 3, total: 5, cost: 0.01 }), "utf-8"); + writeFileSync( + file, + JSON.stringify({ input: 2, output: 3, total: 5, cost: 0.01 }), + "utf-8", + ); - assert.deepEqual(await readRunUsage({}, file, false), { input: 2, output: 3, total: 5, cost: 0.01 }); + assert.deepEqual(await readRunUsage({}, file, false), { + input: 2, + output: 3, + total: 5, + cost: 0.01, + }); + }); + + it("rejects malformed usage writeback instead of leaking it across the run contract", async () => { + const dir = mkdtempSync(join(tmpdir(), "agenta-usage-test-")); + dirs.push(dir); + const file = join(dir, "usage.json"); + writeFileSync( + file, + JSON.stringify({ input: "2", output: 3, total: 5, cost: -1 }), + "utf-8", + ); + + assert.equal(await readRunUsage({}, file, false), undefined); }); it("reads Daytona Pi usage writeback through the sandbox fs API", async () => { const sandbox = { - readFsFile: async () => JSON.stringify({ input: 1, output: 4, total: 5, cost: 0 }), + readFsFile: async () => + JSON.stringify({ input: 1, output: 4, total: 5, cost: 0 }), }; assert.deepEqual(await readRunUsage(sandbox, "/tmp/usage.json", true), { @@ -66,7 +90,11 @@ describe("resolveRunUsage", () => { const dir = mkdtempSync(join(tmpdir(), "agenta-usage-test-")); dirs.push(dir); const file = join(dir, "usage.json"); - writeFileSync(file, JSON.stringify({ input: 3, output: 4, total: 7, cost: 0.03 }), "utf-8"); + writeFileSync( + file, + JSON.stringify({ input: 3, output: 4, total: 7, cost: 0.03 }), + "utf-8", + ); assert.deepEqual( await resolveRunUsage({ diff --git a/services/runner/tests/unit/session-keepalive-approval.test.ts b/services/runner/tests/unit/session-keepalive-approval.test.ts index b24ebf2049..9f9a5efed2 100644 --- a/services/runner/tests/unit/session-keepalive-approval.test.ts +++ b/services/runner/tests/unit/session-keepalive-approval.test.ts @@ -23,10 +23,8 @@ import { type KeepaliveContext, type KeepaliveEngine, } from "../../src/server.ts"; -import { - SessionPool, - type KeepaliveConfig, -} from "../../src/engines/sandbox_agent/session-pool.ts"; +import type { KeepaliveConfig } from "../../src/engines/sandbox_agent/session-identity.ts"; +import { SessionPool } from "../../src/engines/sandbox_agent/session-pool.ts"; import type { MountCredentials } from "../../src/engines/sandbox_agent/mount.ts"; import { acquireEnvironment, diff --git a/services/runner/tests/unit/session-keepalive-dispatch.test.ts b/services/runner/tests/unit/session-keepalive-dispatch.test.ts index 7c3dcb58ca..0067c62423 100644 --- a/services/runner/tests/unit/session-keepalive-dispatch.test.ts +++ b/services/runner/tests/unit/session-keepalive-dispatch.test.ts @@ -23,10 +23,8 @@ import { type KeepaliveContext, type KeepaliveEngine, } from "../../src/server.ts"; -import { - SessionPool, - type KeepaliveConfig, -} from "../../src/engines/sandbox_agent/session-pool.ts"; +import type { KeepaliveConfig } from "../../src/engines/sandbox_agent/session-identity.ts"; +import { SessionPool } from "../../src/engines/sandbox_agent/session-pool.ts"; import type { MountCredentials } from "../../src/engines/sandbox_agent/mount.ts"; import type { SessionEnvironment } from "../../src/engines/sandbox_agent.ts"; @@ -246,7 +244,11 @@ describe("runWithKeepalive: park + hit", () => { makeCtx(daytona.engine), ); - assert.equal(result.ok, true, "the session is already parked; the hook is best-effort"); + assert.equal( + result.ok, + true, + "the session is already parked; the hook is best-effort", + ); }); it("does not call the live-park hook when Daytona park overflows", async () => { @@ -257,11 +259,9 @@ describe("runWithKeepalive: park + hit", () => { approvalTtlMs: 60_000, poolMax: 1, }; - const pool = new SessionPool( - { poolMax: 1 }, - () => {}, - { strictCapacity: true }, - ); + const pool = new SessionPool({ poolMax: 1 }, () => {}, { + strictCapacity: true, + }); const context = { engine, pool, config }; await runWithKeepalive( { ...turn1("occupied"), sandbox: "daytona" }, @@ -608,9 +608,19 @@ describe("runWithKeepalive: never-park rules", () => { }); it("dispatches only to an enabled provider pool", () => { - const disabled = { enabled: false, ttlMs: 0, approvalTtlMs: 0, poolMax: 20 }; + const disabled = { + enabled: false, + ttlMs: 0, + approvalTtlMs: 0, + poolMax: 20, + }; const enabled = { ...disabled, enabled: true, ttlMs: 120_000 }; - const local = { enabled: true, ttlMs: 60_000, approvalTtlMs: 300_000, poolMax: 8 }; + const local = { + enabled: true, + ttlMs: 60_000, + approvalTtlMs: 300_000, + poolMax: 8, + }; assert.equal( resolveKeepaliveDispatch( { sandbox: "daytona" }, @@ -626,7 +636,10 @@ describe("runWithKeepalive: never-park rules", () => { "daytona", ); assert.equal( - resolveKeepaliveDispatch({ sandbox: "local" }, { local, daytona: enabled }), + resolveKeepaliveDispatch( + { sandbox: "local" }, + { local, daytona: enabled }, + ), "local", ); assert.equal( diff --git a/services/runner/tests/unit/session-pool.test.ts b/services/runner/tests/unit/session-pool.test.ts index 9086e52cf6..dca1f77edb 100644 --- a/services/runner/tests/unit/session-pool.test.ts +++ b/services/runner/tests/unit/session-pool.test.ts @@ -20,10 +20,10 @@ import { priorConversation, readKeepaliveConfig, resolvesToLocalProvider, - SessionPool, tailIsFreshUserMessage, type CredentialEpoch, -} from "../../src/engines/sandbox_agent/session-pool.ts"; +} from "../../src/engines/sandbox_agent/session-identity.ts"; +import { SessionPool } from "../../src/engines/sandbox_agent/session-pool.ts"; describe("resolvesToLocalProvider (local/remote gate)", () => { it("is true when the request explicitly asks for local", () => { @@ -601,11 +601,9 @@ describe("SessionPool", () => { stoppingEnv.state.reasons.push(reason); }, }; - const pool = new SessionPool( - { poolMax: 1 }, - () => {}, - { strictCapacity: true }, - ); + const pool = new SessionPool({ poolMax: 1 }, () => {}, { + strictCapacity: true, + }); await pool.park(parkInput("a", stoppingEnv).input, 10_000); const replacement = parkInput("b"); @@ -620,17 +618,19 @@ describe("SessionPool", () => { releaseTeardown?.(); assert.equal(await parked, true); - assert.equal(teardownCompleted, true, "teardown completes before park resolves"); + assert.equal( + teardownCompleted, + true, + "teardown completes before park resolves", + ); assert.equal(pool.get("a"), undefined); assert.equal(pool.get("b")?.state, "idle"); }); it("strict capacity returns false at cap when no idle entry exists", async () => { - const pool = new SessionPool( - { poolMax: 1 }, - () => {}, - { strictCapacity: true }, - ); + const pool = new SessionPool({ poolMax: 1 }, () => {}, { + strictCapacity: true, + }); const busy = parkInput("busy"); await pool.park(busy.input, 10_000); pool.checkoutIdle("busy"); @@ -643,16 +643,10 @@ describe("SessionPool", () => { }); it("strict approval checkout stays seated while it is busy", async () => { - const pool = new SessionPool( - { poolMax: 1 }, - () => {}, - { strictCapacity: true }, - ); - await pool.park( - parkInput("approval").input, - 10_000, - "awaiting_approval", - ); + const pool = new SessionPool({ poolMax: 1 }, () => {}, { + strictCapacity: true, + }); + await pool.park(parkInput("approval").input, 10_000, "awaiting_approval"); const live = pool.checkoutApproval("approval"); @@ -672,11 +666,9 @@ describe("SessionPool", () => { releaseTeardown = resolve; }), }; - const pool = new SessionPool( - { poolMax: 1 }, - () => {}, - { strictCapacity: true }, - ); + const pool = new SessionPool({ poolMax: 1 }, () => {}, { + strictCapacity: true, + }); await pool.park(parkInput("a", environment).input, 10_000); const stopping = pool.get("a")!; const replacement = pool.park(parkInput("b").input, 10_000); @@ -686,14 +678,22 @@ describe("SessionPool", () => { assert.equal(pool.checkoutIdle("a"), undefined); assert.equal(pool.checkoutApproval("a"), undefined); assert.equal( - await pool.repark(stopping, { - configFingerprint: "new", - historyFingerprint: "new", - credentialEpoch: epoch, - }, 10_000), + await pool.repark( + stopping, + { + configFingerprint: "new", + historyFingerprint: "new", + credentialEpoch: epoch, + }, + 10_000, + ), false, ); - assert.equal(pool.get("a"), stopping, "repark does not clobber the seated stop"); + assert.equal( + pool.get("a"), + stopping, + "repark does not clobber the seated stop", + ); releaseTeardown?.(); assert.equal(await replacement, true);