From 20aa346bd1960d38499ec112406ec1d78c0dbd3c Mon Sep 17 00:00:00 2001 From: Chris Huber Date: Tue, 14 Jul 2026 13:21:12 -0400 Subject: [PATCH] Canonicalize reviewer evidence before sanitization --- .../execute-native-agent-task.mjs | 59 ++++++++++++++++++- .../prepare-agent-task-upload.mjs | 45 ++++++++------ ...e-native-agent-task-playground-e2e.test.ts | 3 +- tests/runtime-sources-materialization.test.ts | 16 ++++- 4 files changed, 99 insertions(+), 24 deletions(-) diff --git a/.github/scripts/run-agent-task/execute-native-agent-task.mjs b/.github/scripts/run-agent-task/execute-native-agent-task.mjs index bf31c5c7..25387a23 100644 --- a/.github/scripts/run-agent-task/execute-native-agent-task.mjs +++ b/.github/scripts/run-agent-task/execute-native-agent-task.mjs @@ -1,8 +1,9 @@ import { rmSync } from "node:fs" -import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises" +import { appendFile, lstat, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises" import { isAbsolute, join, relative, resolve } from "node:path" import { pathToFileURL } from "node:url" import { spawn } from "node:child_process" +import { createHash } from "node:crypto" import { canonicalExternalNativeAgentIdentity, materializeExternalNativePackage, materializeRuntimeSources, normalizeExternalPackageSource, normalizeRuntimeSources, parseExternalPackageSourcePolicy, sha256BytesV1, validateRuntimeSourceModel } from "./materialize-external-native-package.mjs" import { readNativeResult } from "./native-result-file.mjs" import { assertNoRuntimeSourcePaths, sanitizeRuntimeSourceJson, sanitizeRuntimeSourceText, sanitizeRuntimeSourceValue } from "./runtime-source-sanitizer.mjs" @@ -20,6 +21,7 @@ const secretValues = ["OPENAI_API_KEY", "MODEL_PROVIDER_SECRET_1", "MODEL_PROVID let privateRuntimeSourceRoot = "" let privateRuntimeSourceRootForSanitization = "" let runnerWorkspaceSeedSnapshot +let reviewerEvidence const SIGNAL_EXIT_CODES = { SIGINT: 130, SIGTERM: 143, SIGHUP: 129 } let materializedSourceCleanup @@ -312,6 +314,7 @@ async function writeNormalizedFailure(error, request = {}) { success: false, request_path: workflowPath(requestPath), failure: { ...failure, message }, + ...(reviewerEvidence ? { reviewer_evidence: reviewerEvidence } : {}), ...(accessError ? { access: { authorized: false, error: message } } : {}), } await mkdir(join(workspace, ".codebox"), { recursive: true }) @@ -322,11 +325,59 @@ async function writeNormalizedFailure(error, request = {}) { await output("result_path", ".codebox/agent-task-workflow-result.json") } -async function redactArtifactFiles(directory) { +function underRoot(root, path) { + const contained = relative(root, path) + return contained !== ".." && !contained.startsWith(`..${String.fromCharCode(47)}`) && !isAbsolute(contained) +} + +async function canonicalReviewerTranscript(nativeRuntimeResult, artifactsPath) { + const publicCore = await import(pathToFileURL(join(codeboxRoot, "packages/runtime-core/dist/public.js")).href) + const refs = publicCore.normalizePublicArtifactRefDTOs(nativeRuntimeResult) + .filter((ref) => ref.kind === "codebox-transcript" && typeof ref.path === "string" && ref.path) + if (refs.length === 0) return undefined + + const root = await realpath(artifactsPath) + const artifactRoot = resolve(artifactsPath) + const transcripts = new Map() + for (const ref of refs) { + // Resolve first so containment, rather than spelling, defines a trusted path. + const requested = resolve(artifactRoot, ref.path) + const canonical = await realpath(requested).catch((error) => error?.code === "ENOENT" ? "" : Promise.reject(error)) + if (!canonical) continue + if (!underRoot(root, canonical)) throw new Error("Canonical transcript escapes the trusted artifact root.") + const requestedRelative = relative(artifactRoot, requested) + if (!underRoot(artifactRoot, requested)) throw new Error("Canonical transcript escapes the trusted artifact root.") + let current = artifactRoot + for (const part of requestedRelative.split("/").filter(Boolean)) { + current = join(current, part) + const metadata = await lstat(current) + if (metadata.isSymbolicLink()) throw new Error("Canonical transcript must not traverse symlinks.") + } + const metadata = await lstat(current) + if (!metadata.isFile()) throw new Error("Canonical transcript must be a regular file.") + const source = await realpath(current) + const bytes = await readFile(source) + const raw = JSON.parse(bytes.toString("utf8")) + if (raw?.schema !== "wp-codebox/agent-transcript/v1") throw new Error("Canonical transcript must use wp-codebox/agent-transcript/v1.") + transcripts.set(source, { + schema: raw.schema, + kind: "codebox-transcript", + path: relative(root, source).replaceAll("\\", "/"), + source_sha256: createHash("sha256").update(bytes).digest("hex"), + size_bytes: bytes.length, + }) + } + if (transcripts.size === 0) return undefined + if (transcripts.size !== 1) throw new Error("Canonical transcript requires exactly one distinct existing file.") + return { transcript: [...transcripts.values()][0] } +} + +async function redactArtifactFiles(directory, artifactRoot = directory) { const { readdir, stat } = await import("node:fs/promises") for (const entry of await readdir(directory, { withFileTypes: true })) { const path = join(directory, entry.name) - if (entry.isDirectory()) await redactArtifactFiles(path) + if (reviewerEvidence?.transcript && path === resolve(artifactRoot, reviewerEvidence.transcript.path)) continue + if (entry.isDirectory()) await redactArtifactFiles(path, artifactRoot) if (entry.isFile() && path.endsWith(".json") && (await stat(path)).size <= 4 * 1024 * 1024) { const contents = await readFile(path, "utf8").catch(() => null) if (contents !== null) { @@ -467,6 +518,7 @@ const nativeRuntimeResult = request.run_agent && !request.dry_run ? await readNativeResult(nativeResultPath, controlledCodeboxPath, secretValues, redact) : {} await rm(nativeResultPath, { force: true }) +reviewerEvidence = await canonicalReviewerTranscript(nativeRuntimeResult, artifactsPath) let runtimeResult = sanitizeRuntimeSourceValue(nativeRuntimeResult, privateRuntimeSourceRootForSanitization) assertNoRuntimeSourcePaths(runtimeResult, privateRuntimeSourceRootForSanitization) let workspaceApply = { status: "no-op", changedFiles: [] } @@ -559,6 +611,7 @@ const result = { runtime_input_path: ".codebox/native-agent-task-input.json", execution: { stdout_truncated: execution.stdout_truncated, stderr_truncated: execution.stderr_truncated }, runtime_result: redact(runtimeRecord), + ...(reviewerEvidence ? { reviewer_evidence: reviewerEvidence } : {}), verification, publication, transcript: { artifact_name: request.artifacts?.transcript_name || "agent-task-transcript" }, diff --git a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs index 2e070ee2..f3b9fa89 100644 --- a/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs +++ b/.github/scripts/run-agent-task/prepare-agent-task-upload.mjs @@ -2,7 +2,6 @@ import { constants } from "node:fs" import { lstat, mkdir, open, readdir, readFile, realpath, rm, writeFile } from "node:fs/promises" import { isUtf8 } from "node:buffer" import { createHash } from "node:crypto" -import { fileURLToPath, pathToFileURL } from "node:url" import { isAbsolute, join, relative, resolve } from "node:path" import { assertNoRuntimeSourcePaths, sanitizeRuntimeSourceJson } from "./runtime-source-sanitizer.mjs" @@ -18,7 +17,6 @@ const runtimeSourceRoot = process.env.WP_CODEBOX_RUNTIME_SOURCE_ROOT ? resolve(p const runtimeSourcePrefix = process.env.WP_CODEBOX_RUNTIME_SOURCE_PREFIX ? resolve(process.env.WP_CODEBOX_RUNTIME_SOURCE_PREFIX) : "" const runtimeSourceRoots = [runtimeSourceRoot, runtimeSourcePrefix].filter(Boolean) const privateUploadRoots = [...runtimeSourceRoots, workspace] -const codeboxRoot = resolve(fileURLToPath(new URL("../../..", import.meta.url))) const SOURCE_TREE = /(^|\/)(prepared-plugins|prepared-source-packages|source-package[^/]*)(\/|$)/i const SOURCE_FILE = /\.(?:php|phtml|js|mjs|cjs|jsx|ts|tsx)$/i const PHP_OPENING_TAG = /<\?(?:php|=)(?:\s|$)/i @@ -147,13 +145,21 @@ async function stageTextFile(source, destination, options = {}) { return true } -async function canonicalTranscript(result) { - const publicCore = await import(pathToFileURL(join(codeboxRoot, "packages/runtime-core/dist/public.js")).href) - const refs = publicCore.normalizePublicArtifactRefDTOs(record(result).runtime_result) - .filter((ref) => ref.kind === "codebox-transcript") - if (refs.length === 0) return undefined - if (refs.length !== 1 || !safeRelativeArtifactPath(refs[0].path)) throw new Error("Canonical transcript requires exactly one trusted codebox-transcript path.") - return refs[0] +function canonicalTranscript(result) { + const descriptor = record(record(result).reviewer_evidence).transcript + if (descriptor === undefined) return undefined + const transcript = record(descriptor) + if (Object.keys(transcript).length !== 5 + || transcript.schema !== "wp-codebox/agent-transcript/v1" + || transcript.kind !== "codebox-transcript" + || typeof transcript.path !== "string" + || !/^[a-f0-9]{64}$/.test(transcript.source_sha256) + || !Number.isSafeInteger(transcript.size_bytes) + || transcript.size_bytes < 0 + || transcript.size_bytes > MAX_UPLOAD_FILE_BYTES) { + throw new Error("Reviewer evidence transcript descriptor is malformed.") + } + return transcript } function digest(bytes) { @@ -214,7 +220,12 @@ function projectParsed(value) { async function trustedTranscriptFile(path) { const root = await realpath(artifactsPath).catch(() => "") if (!root) return { unavailable: "artifact-root-missing" } - const parts = path.split("/") + // Resolve before evaluating containment so harmless relative spelling is not + // mistaken for an escape while aliases and symlinks still fail closed. + const requested = resolve(root, path) + const requestedRelative = relative(root, requested) + if (requestedRelative === ".." || requestedRelative.startsWith(`..${String.fromCharCode(47)}`) || isAbsolute(requestedRelative)) throw new Error("Canonical transcript escapes the trusted artifact root.") + const parts = requestedRelative.split("/").filter(Boolean) let current = root for (const part of parts) { current = join(current, part) @@ -233,18 +244,16 @@ async function trustedTranscriptFile(path) { async function stageCanonicalTranscript(result) { const ref = await canonicalTranscript(result) if (!ref) return undefined - const path = safeRelativeArtifactPath(ref.path) - if (sourceCategory(path, resolve(artifactsPath, path))) throw new Error("Canonical transcript must stay under the trusted artifact root.") - const trusted = await trustedTranscriptFile(path) - if (trusted.unavailable) return { unavailable: trusted.unavailable, provenance: { kind: ref.kind, artifact_path: path } } + const trusted = await trustedTranscriptFile(ref.path) + if (trusted.unavailable) return { unavailable: trusted.unavailable, provenance: { kind: ref.kind, artifact_path: ref.path } } const source = trusted.source const bytes = await readFile(source) if (bytes.includes(0) || !isUtf8(bytes)) throw new Error("Canonical transcript must be UTF-8 JSON.") const actualDigest = digest(bytes) - const expectedDigest = typeof ref.sha256 === "string" ? ref.sha256 : record(ref.digest).value - if (expectedDigest && expectedDigest !== actualDigest) throw new Error("Canonical transcript digest does not match its normalized ref.") + if (ref.source_sha256 !== actualDigest) throw new Error("Canonical transcript digest does not match its reviewer evidence descriptor.") + if (ref.size_bytes !== bytes.length) throw new Error("Canonical transcript size does not match its reviewer evidence descriptor.") const raw = parseJsonOrEmpty(bytes.toString("utf8")) - if (raw.schema !== "wp-codebox/agent-transcript/v1" || !Array.isArray(raw.executions) || raw.executions.length > MAX_TRANSCRIPT_EXECUTIONS) throw new Error("Canonical transcript must be a bounded wp-codebox/agent-transcript/v1 envelope.") + if (raw.schema !== ref.schema || !Array.isArray(raw.executions) || raw.executions.length > MAX_TRANSCRIPT_EXECUTIONS) throw new Error("Canonical transcript must be a bounded wp-codebox/agent-transcript/v1 envelope.") const projection = { schema: "wp-codebox/reviewer-agent-transcript/v1", executions: raw.executions.map((execution, index) => { @@ -260,7 +269,7 @@ async function stageCanonicalTranscript(result) { return { path: ".codebox/agent-task-artifacts/transcript.json", sha256: projectionDigest, - provenance: { kind: ref.kind, artifact_path: path, source_sha256: actualDigest, projection_sha256: projectionDigest }, + provenance: { kind: ref.kind, artifact_path: ref.path, source_sha256: actualDigest, projection_sha256: projectionDigest }, } } diff --git a/tests/execute-native-agent-task-playground-e2e.test.ts b/tests/execute-native-agent-task-playground-e2e.test.ts index 98667c0f..ee9b88b9 100644 --- a/tests/execute-native-agent-task-playground-e2e.test.ts +++ b/tests/execute-native-agent-task-playground-e2e.test.ts @@ -87,7 +87,8 @@ add_filter( 'pre_http_request', static function( $preempt, $args, $url ) { for (const privateValue of ["e2e-github-secret", "e2e-openai-secret", "PRIVATE_WORKSPACE_SENTINEL", "PRIVATE_NPM_SENTINEL", "PRIVATE_NETRC_SENTINEL", "PRIVATE_KEY_SENTINEL", "PRIVATE_CODEBOX_SENTINEL", "PRIVATE_NODE_MODULES_SENTINEL", sources.root, workspace]) assert.doesNotMatch(serialized, new RegExp(privateValue.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))) await execFileAsync(process.execPath, [uploader], { cwd: workspace, env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_REQUEST_PATH: join(workspace, ".codebox", "agent-task-request.json"), AGENT_TASK_UPLOAD_PATH: join(workspace, ".codebox", "agent-task-upload"), GITHUB_TOKEN: "e2e-github-secret", OPENAI_API_KEY: "e2e-openai-secret" } }) const exclusions = JSON.parse(await readFile(join(workspace, ".codebox", "agent-task-upload", ".codebox", "agent-task-artifacts", "exclusions.json"), "utf8")) - assert.equal(exclusions.canonical_transcripts[0].unavailable, "referenced-file-missing", "missing canonical evidence remains optional") + assert.equal(exclusions.canonical_transcripts.length, 1, "available canonical evidence is projected once") + assert.match(exclusions.canonical_transcripts[0].provenance.artifact_path, /^runtime-[a-z0-9-]+\/files\/transcript\.json$/) const staged = JSON.stringify(await Promise.all((await readdir(join(workspace, ".codebox", "agent-task-upload"), { recursive: true })).map((path) => readFile(join(workspace, ".codebox", "agent-task-upload", path), "utf8").catch(() => "")))) for (const privateValue of ["wp-codebox-runner-workspace-seed-", workspace, "PRIVATE_WORKSPACE_SENTINEL", "PRIVATE_NPM_SENTINEL", "PRIVATE_NETRC_SENTINEL", "PRIVATE_KEY_SENTINEL"]) assert.doesNotMatch(staged, new RegExp(privateValue.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))) const stagedInput = JSON.parse(await readFile(join(workspace, ".codebox", "agent-task-upload", ".codebox", "native-agent-task-input.json"), "utf8")) diff --git a/tests/runtime-sources-materialization.test.ts b/tests/runtime-sources-materialization.test.ts index 54a74b66..3de522e7 100644 --- a/tests/runtime-sources-materialization.test.ts +++ b/tests/runtime-sources-materialization.test.ts @@ -169,7 +169,8 @@ await withTempDir("wp-codebox-runtime-source-upload-", async (directory) => { const transcriptSource = JSON.stringify({ schema: "wp-codebox/agent-transcript/v1", executions: [{ executionIndex: 0, command: "wp-codebox.agent-sandbox-run", exitCode: 1, stderr: "Repeated workspace error", parsed: { agent: { id: "fixture", provider: "openai" }, messages: [{ role: "assistant", content: "Target snippet: { const transcriptExclusions = await readFile(join(upload, ".codebox", "agent-task-artifacts", "exclusions.json"), "utf8") assert.match(transcriptExclusions, /canonical_transcripts/) assert.match(transcriptExclusions, /codebox-transcript/) + await writeFile(join(artifacts, "files", "transcript.json"), `${transcriptSource}\n`) + await assert.rejects(execFileAsync(process.execPath, [script.pathname], { env: { ...process.env, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_UPLOAD_PATH: upload, WP_CODEBOX_RUNTIME_SOURCE_ROOT: privateRoot } }), /digest does not match/, "Uploader rejects stale reviewer evidence digests") + await writeFile(join(artifacts, "files", "transcript.json"), transcriptSource) const outsideTranscript = join(directory, "outside-transcript.json") await writeFile(outsideTranscript, JSON.stringify({ secret: "outside" })) await rm(join(artifacts, "files", "transcript.json")) @@ -245,7 +249,7 @@ await withTempDir("wp-codebox-runtime-sources-workflow-", async (directory) => { await writeFile(join(tools, "git"), `#!${process.execPath}\nimport { spawnSync } from "node:child_process"\nconst args = process.argv.slice(2).map((value) => value.startsWith("https://github.com/") ? ${JSON.stringify(repository)} : value)\nconst result = spawnSync(${JSON.stringify(gitPath)}, args, { stdio: "inherit" })\nprocess.exit(result.status ?? 1)\n`) await chmod(join(tools, "git"), 0o755) const capturedInput = join(directory, "captured-native-input.json") - await writeFile(join(directory, "fake-cli.mjs"), `import { readFile, writeFile } from "node:fs/promises"\nconst input = process.argv[process.argv.indexOf("--input-file") + 1]\nconst result = process.argv[process.argv.indexOf("--result-file") + 1]\nconst taskInput = JSON.parse(await readFile(input))\nconst runtimeRoot = taskInput.source_package_root.replace(/\\/prepared-runtime-sources$/, "")\nconst nativeResult = JSON.parse(${JSON.stringify(JSON.stringify(hostedPathRegression.success))}.replaceAll(${JSON.stringify(hostedPathRegression.runtime_root)}, runtimeRoot))\nnativeResult.status = "no_op"\nnativeResult.agent_task_run_result.status = "no_op"\nawait writeFile(${JSON.stringify(capturedInput)}, JSON.stringify(taskInput))\nawait writeFile(result, JSON.stringify(nativeResult))\n`) + await writeFile(join(directory, "fake-cli.mjs"), `import { readFile, writeFile } from "node:fs/promises"\nconst input = process.argv[process.argv.indexOf("--input-file") + 1]\nconst result = process.argv[process.argv.indexOf("--result-file") + 1]\nconst taskInput = JSON.parse(await readFile(input))\nconst runtimeRoot = taskInput.source_package_root.replace(/\\/prepared-runtime-sources$/, "")\nconst nativeResult = JSON.parse(${JSON.stringify(JSON.stringify(hostedPathRegression.success))}.replaceAll(${JSON.stringify(hostedPathRegression.runtime_root)}, runtimeRoot))\nnativeResult.status = "no_op"\nnativeResult.agent_task_run_result.status = "no_op"\nconst transcriptPath = taskInput.artifacts_path + "/files/transcript.json"\nconst transcriptPaths = process.env.GITHUB_TOKEN === "distinct-token" ? [transcriptPath, taskInput.artifacts_path + "/files/transcript-2.json"] : Array.from({ length: 16 }, () => transcriptPath)\nnativeResult.agent_task_run_result.refs = { transcripts: transcriptPaths.map((path) => ({ kind: "codebox-transcript", path })) }\nawait writeFile(${JSON.stringify(capturedInput)}, JSON.stringify(taskInput))\nawait writeFile(result, JSON.stringify(nativeResult))\n`) const request = { workload: { id: "runtime-sources-workflow", label: "Runtime sources workflow" }, access: { caller_repo: "example/target", allowed_repos: ["example/target"], access_token_repos: ["example/target"] }, @@ -265,11 +269,19 @@ await withTempDir("wp-codebox-runtime-sources-workflow-", async (directory) => { } await writeFile(join(codebox, "agent-task-request.json"), JSON.stringify(request)) await writeFile(join(codebox, "agent-task-artifacts", "safe.json"), JSON.stringify({ provenance: { repository: "example/source", revision } })) + await mkdir(join(codebox, "agent-task-artifacts", "files"), { recursive: true }) + await writeFile(join(codebox, "agent-task-artifacts", "files", "transcript.json"), JSON.stringify({ schema: "wp-codebox/agent-transcript/v1", executions: [{ command: "fixture", exitCode: 0 }] })) const githubOutput = join(directory, "github-output") const environment = { ...process.env, PATH: `${tools}:${process.env.PATH}`, TMPDIR: temp, GITHUB_OUTPUT: githubOutput, AGENT_TASK_WORKSPACE: workspace, AGENT_TASK_REQUEST_PATH: join(codebox, "agent-task-request.json"), WP_CODEBOX_CLI_PATH: join(directory, "fake-cli.mjs"), GITHUB_TOKEN: "test-token", EXTERNAL_PACKAGE_SOURCE_POLICY: JSON.stringify({ version: 1, repositories: { "example/source": ["fixture.agent.json"] }, runtime_sources: { "example/source": ["plugin"] } }) } const executorPath = new URL("../.github/scripts/run-agent-task/execute-native-agent-task.mjs", import.meta.url) await execFileAsync(process.execPath, [executorPath.pathname], { env: environment }) const workflowResult = await readFile(join(codebox, "agent-task-workflow-result.json"), "utf8") + const canonicalizedResult = JSON.parse(workflowResult) + assert.deepEqual(canonicalizedResult.reviewer_evidence, { transcript: { schema: "wp-codebox/agent-transcript/v1", kind: "codebox-transcript", path: "files/transcript.json", source_sha256: createHash("sha256").update(await readFile(join(codebox, "agent-task-artifacts", "files", "transcript.json"))).digest("hex"), size_bytes: (await readFile(join(codebox, "agent-task-artifacts", "files", "transcript.json"))).length } }, "hosted duplicate absolute refs reduce to one reviewer evidence descriptor") + await execFileAsync(process.execPath, [new URL("../.github/scripts/run-agent-task/prepare-agent-task-upload.mjs", import.meta.url).pathname], { env: { ...environment, AGENT_TASK_UPLOAD_PATH: upload } }) + assert.ok(await readFile(join(upload, ".codebox", "agent-task-artifacts", "transcript.json"), "utf8"), "uploader uses the dedicated canonical descriptor") + await writeFile(join(codebox, "agent-task-artifacts", "files", "transcript-2.json"), JSON.stringify({ schema: "wp-codebox/agent-transcript/v1", executions: [] })) + await assert.rejects(execFileAsync(process.execPath, [executorPath.pathname], { env: { ...environment, GITHUB_TOKEN: "distinct-token" } }), /exactly one distinct existing file/, "distinct transcript files fail closed") const exactRuntimeRoot = JSON.parse(await readFile(capturedInput, "utf8")).source_package_root.replace(/\/prepared-runtime-sources$/, "") assert.doesNotMatch(workflowResult, new RegExp(exactRuntimeRoot.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))) assert.doesNotMatch(workflowResult, /source_package_root/)