Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 92 additions & 0 deletions .github/scripts/run-agent-task/agent-runtime-failure.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/**
* Explicit agent-runtime failure for the agent-task workflow result.
*
* When the native agent runtime exits non-zero or reports `success !== true`,
* the workflow result must carry a stage/code/message explaining why. Without
* this projection a failed runtime surfaced as `success: false` with no
* `failure` object at all (the raw runtime result is not part of the uploaded
* reviewer projection), so a failed run carried no reason.
*
* Kept free of side effects so it is unit-testable without running the
* executor entrypoint.
*/

const MAX_DIAGNOSTICS = 5
const MAX_TEXT = 2048

function record(value) {
return value && typeof value === "object" && !Array.isArray(value) ? value : {}
}

function text(value, limit = MAX_TEXT) {
if (typeof value !== "string") return ""
const trimmed = value.trim()
return trimmed.length > limit ? `${trimmed.slice(0, limit)} [TRUNCATED]` : trimmed
}

function diagnosticList(...sources) {
const seen = new Set()
const diagnostics = []
for (const source of sources) {
for (const entry of Array.isArray(source) ? source : []) {
const diagnostic = record(entry)
const code = text(diagnostic.code, 256)
const message = text(diagnostic.message)
if (!code && !message) continue
const key = `${code}\u0000${message}`
if (seen.has(key)) continue
seen.add(key)
diagnostics.push(Object.fromEntries(Object.entries({ code: code || undefined, message: message || undefined, severity: text(diagnostic.severity, 32) || undefined }).filter(([, item]) => item !== undefined)))
}
}
// Errors first so the headline names the actual cause, not a warning.
return diagnostics.sort((a, b) => Number(b.severity === "error") - Number(a.severity === "error"))
}

/**
* Projects why the agent runtime did not succeed. Returns undefined when the
* runtime exited zero and reported success.
*/
export function agentRuntimeFailure({ execution, runtimeResult, bound = (value) => text(value, 8192) }) {
const exitCode = Number.isSafeInteger(execution?.code) ? execution.code : undefined
const runtime = record(runtimeResult)
if (exitCode === 0 && runtime.success === true) return undefined

const summary = record(runtime.agent_task_run_result)
const terminal = record(runtime.terminal_result ?? summary.terminal_result)
const providerError = record(record(summary.metadata).provider_error)
const diagnostics = diagnosticList(runtime.diagnostics, summary.diagnostics, terminal.diagnostics)
const status = text(summary.status, 64) || text(runtime.status, 64)
const terminalStatus = text(terminal.status, 64)
const classification = text(summary.failure_classification, 64)
|| text(terminal.failure_classification, 64)
|| (Object.keys(providerError).length ? "provider_error" : "")
|| "agent_runtime"
const headline = diagnostics[0]
const stderrTail = bound(typeof execution?.stderr === "string" ? execution.stderr : "")
const stdoutTail = bound(typeof execution?.stdout === "string" ? execution.stdout : "")
const code = headline?.code
|| (terminalStatus && terminalStatus !== "succeeded" ? `agent_${terminalStatus}` : "")
|| (status && status !== "succeeded" ? `agent_${status}` : "")
|| (exitCode !== undefined && exitCode !== 0 ? "agent_runtime_exit" : "agent_runtime_failed")
const providerMessage = text(providerError.message)
const message = headline?.message
|| providerMessage
|| (exitCode !== undefined && exitCode !== 0
? `The agent runtime exited with code ${exitCode}${status ? ` (status: ${status})` : ""}.${stderrTail ? ` stderr: ${text(stderrTail, 512)}` : ""}`
: `The agent runtime did not report success${status ? ` (status: ${status})` : ""}.`)

return Object.fromEntries(Object.entries({
stage: "agent",
code,
classification,
message,
exit_code: exitCode,
runtime_status: status || undefined,
terminal_status: terminalStatus || undefined,
diagnostics: diagnostics.length ? diagnostics.slice(0, MAX_DIAGNOSTICS) : undefined,
provider_error: providerMessage ? { message: providerMessage, ...(text(providerError.code, 128) ? { code: text(providerError.code, 128) } : {}) } : undefined,
stdout_tail: exitCode !== 0 && stdoutTail ? stdoutTail : undefined,
stderr_tail: exitCode !== 0 && stderrTail ? stderrTail : undefined,
}).filter(([, item]) => item !== undefined))
}
41 changes: 38 additions & 3 deletions .github/scripts/run-agent-task/execute-native-agent-task.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { publishRunnerWorkspace } from "./runner-workspace-publisher.mjs"
import { createRunnerWorkspaceSeedSnapshot, RUNNER_WORKSPACE_SEED_EXCLUDES } from "./runner-workspace-seed-snapshot.mjs"
import { runnerWorkspaceApplyOutcome, sandboxWorkspaceMutations } from "./runner-workspace-apply-outcome.mjs"
import { agentTurnLimitFailure, agentTurnLimitOutcome } from "./agent-turn-limit-outcome.mjs"
import { agentRuntimeFailure } from "./agent-runtime-failure.mjs"
import { createTrustedArtifactApplyChannel, trustedArtifactApplyRefs } from "./trusted-artifact-snapshot.mjs"
import { artifactSourcePathCategory, assertNoSeedSnapshotPaths, containsRuntimeSourceContent, sanitizeSeedSnapshotJson } from "./artifact-upload-policy.mjs"
import { publishProcessMarker } from "../../../scripts/process-marker.mjs"
Expand All @@ -24,6 +25,9 @@ const outputPath = process.env.GITHUB_OUTPUT
const MAX_CAPTURE_BYTES = 32768
const MAX_WORKFLOW_OUTPUT_BYTES = 8192
const MAX_COMMAND_ARTIFACT_BYTES = 4 * 1024 * 1024
// Long agent runs (40+ turns, parallel tool calls) routinely exceed 64 calls;
// dropping the whole projection hid every write from the apply outcome.
const MAX_TOOL_OBSERVABILITY_CALLS = 512
const secretValues = ["OPENAI_API_KEY", "MODEL_PROVIDER_SECRET_1", "MODEL_PROVIDER_SECRET_2", "MODEL_PROVIDER_SECRET_3", "MODEL_PROVIDER_SECRET_4", "MODEL_PROVIDER_SECRET_5", "GITHUB_TOKEN", "GH_TOKEN", "ACCESS_TOKEN", "EXTERNAL_PACKAGE_SOURCE_POLICY"].map((name) => process.env[name]).filter(Boolean)
let privateRuntimeSourceRoot = ""
let privateRuntimeSourceRootForSanitization = ""
Expand Down Expand Up @@ -199,7 +203,7 @@ function record(value) {

function canonicalToolObservability(metadata) {
const source = record(record(record(metadata).agents_api).tool_observability)
if (source.version !== 1 || !Array.isArray(source.calls) || source.calls.length > 64) return undefined
if (source.version !== 1 || !Array.isArray(source.calls) || source.calls.length > MAX_TOOL_OBSERVABILITY_CALLS) return undefined
const calls = source.calls.map(projectCanonicalToolCall).filter(Boolean)
return calls.length ? { version: 1, calls } : undefined
}
Expand Down Expand Up @@ -605,12 +609,37 @@ async function appliedWorkspaceEvidence(refs, artifactsPath) {
return Object.keys(evidence).length === 2 ? evidence : undefined
}

// When the native result is unreadable (or carries no transcript ref) the
// runtime may still have written its canonical transcript. Discover exactly
// one runtime-*/files/transcript.json so failed runs keep reviewer evidence.
async function discoveredTranscriptRefs(artifactsPath) {
const { readdir } = await import("node:fs/promises")
const entries = await readdir(artifactsPath, { withFileTypes: true }).catch(() => [])
const refs = []
for (const entry of entries) {
if (!entry.isDirectory() || entry.isSymbolicLink() || !/^runtime-[A-Za-z0-9._-]+$/.test(entry.name)) continue
const candidate = join(artifactsPath, entry.name, "files", "transcript.json")
const metadata = await lstat(candidate).catch(() => undefined)
if (metadata?.isFile()) refs.push({ kind: "codebox-transcript", path: `${entry.name}/files/transcript.json` })
}
return refs.length === 1 ? refs : []
}

async function canonicalReviewerTranscript(nativeRuntimeResult, artifactsPath) {
const publicCore = await import(pathToFileURL(join(codeboxRoot, "packages/runtime-core/dist/public.js")).href)
const refs = publicCore.normalizePublicArtifactRefDTOs(nativeRuntimeResult)
let refs = publicCore.normalizePublicArtifactRefDTOs(nativeRuntimeResult)
.filter((ref) => ref.kind === "codebox-transcript" && typeof ref.path === "string" && ref.path)
if (refs.length === 0) return undefined
if (refs.length === 0) {
const discovered = await discoveredTranscriptRefs(artifactsPath)
if (discovered.length === 0) return undefined
// Best effort: a discovered transcript that fails validation is skipped
// rather than masking the runtime failure it is meant to explain.
return trustedReviewerTranscript(discovered, artifactsPath).catch(() => undefined)
}
return trustedReviewerTranscript(refs, artifactsPath)
}

async function trustedReviewerTranscript(refs, artifactsPath) {
const root = await realpath(artifactsPath)
const artifactRoot = resolve(artifactsPath)
const transcripts = new Map()
Expand Down Expand Up @@ -822,6 +851,12 @@ const agentTurnLimit = request.run_agent && !request.dry_run
: undefined
const agentTurnCapReached = agentTurnLimit?.status === "max_turns"
if (agentTurnCapReached) downstreamFailure = agentTurnLimitFailure(agentTurnLimit)
// Any other runtime failure (non-zero exit, unreadable native result, provider
// error, failed terminal status) must name its cause; the raw runtime result is
// not part of the uploaded reviewer projection.
if (request.run_agent && !request.dry_run && !downstreamFailure) {
downstreamFailure = agentRuntimeFailure({ execution, runtimeResult, bound: (value) => bounded(value, MAX_WORKFLOW_OUTPUT_BYTES) })
}
if (execution.code === 0 && runtimeResult.success === true && request.runner_workspace?.enabled && !agentTurnCapReached) {
try {
runnerWorkspaceCore = await import(pathToFileURL(join(codeboxRoot, "packages/runtime-core/dist/runner-workspace-apply.js")).href)
Expand Down
6 changes: 4 additions & 2 deletions .github/scripts/run-agent-task/native-result-file.mjs
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import { lstat, readFile } from "node:fs/promises"
import { resolve } from "node:path"

export const MAX_NATIVE_RESULT_BYTES = 1024 * 1024
// The native result embeds the full recipe run (including the agent loop output),
// which grows with every turn; a 1 MiB bound silently discarded long runs.
export const MAX_NATIVE_RESULT_BYTES = 32 * 1024 * 1024
const TERMINAL_STATUSES = new Set(["succeeded", "failed", "no_op", "timeout", "provider_error", "unable_to_remediate"])

export function nativeResultFailure(code, message) {
Expand All @@ -20,7 +22,7 @@ export async function readNativeResult(path, controlledDirectory, secretValues,
return nativeResultFailure("wp-codebox.agent-task.result-file", "Native agent-task result file must be a regular non-symlink file.")
}
if (info.size > MAX_NATIVE_RESULT_BYTES) {
return nativeResultFailure("wp-codebox.agent-task.result-too-large", `Native agent-task result exceeds the ${MAX_NATIVE_RESULT_BYTES}-byte limit.`)
return nativeResultFailure("wp-codebox.agent-task.result-too-large", `Native agent-task result is ${info.size} bytes, exceeding the ${MAX_NATIVE_RESULT_BYTES}-byte limit.`)
}
const contents = await readFile(resolvedPath, "utf8")
if (Buffer.byteLength(contents) !== info.size) {
Expand Down
14 changes: 9 additions & 5 deletions .github/scripts/run-agent-task/prepare-agent-task-upload.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ import { artifactSourcePathCategory, assertNoSeedSnapshotPaths, containsRuntimeS

const MAX_UPLOAD_FILE_BYTES = 4 * 1024 * 1024
const MAX_TRANSCRIPT_EXECUTIONS = 64
// The canonical transcript source is only read to build a bounded projection;
// long agent runs legitimately exceed the 4 MiB per-file upload bound.
const MAX_TRANSCRIPT_SOURCE_BYTES = 32 * 1024 * 1024
const MAX_TOOL_OBSERVABILITY_CALLS = 512
const MAX_REVIEW_TEXT_BYTES = 32 * 1024
const MAX_TOOL_ARGUMENT_KEYS = 32
const MAX_REVIEW_COLLECTION_ENTRIES = 64
Expand Down Expand Up @@ -125,7 +129,7 @@ function canonicalTranscript(result) {
|| !/^[a-f0-9]{64}$/.test(transcript.source_sha256)
|| !Number.isSafeInteger(transcript.size_bytes)
|| transcript.size_bytes < 0
|| transcript.size_bytes > MAX_UPLOAD_FILE_BYTES) {
|| transcript.size_bytes > MAX_TRANSCRIPT_SOURCE_BYTES) {
throw new Error("Reviewer evidence transcript descriptor is malformed.")
}
return transcript
Expand Down Expand Up @@ -244,7 +248,7 @@ function projectToolCall(value) {
// The canonical `wordpress.run-php` execution carries the agent runtime result
// as a JSON-encoded string in `parsed.output`; decode it so tool calls survive.
function decodedParsedOutput(parsed) {
if (typeof parsed.output !== "string" || Buffer.byteLength(parsed.output) > MAX_UPLOAD_FILE_BYTES) return record(parsed.output)
if (typeof parsed.output !== "string" || Buffer.byteLength(parsed.output) > MAX_TRANSCRIPT_SOURCE_BYTES) return record(parsed.output)
try {
return record(JSON.parse(parsed.output))
} catch {
Expand All @@ -257,7 +261,7 @@ function decodedParsedOutput(parsed) {
function projectWorkspaceMutations(value) {
const ledger = record(value)
if (ledger.schema !== "wp-codebox/sandbox-workspace-mutations/v1" || !Array.isArray(ledger.calls)) return undefined
const calls = ledger.calls.slice(0, MAX_TRANSCRIPT_EXECUTIONS).flatMap((value) => {
const calls = ledger.calls.slice(0, MAX_TOOL_OBSERVABILITY_CALLS).flatMap((value) => {
const call = record(value)
if (!["workspace_write", "workspace_edit", "workspace_apply_patch"].includes(call.tool) || !["succeeded", "failed"].includes(call.status)) return []
const paths = (Array.isArray(call.paths) ? call.paths : []).map(safeTargetPath).filter(Boolean).slice(0, 32)
Expand Down Expand Up @@ -295,7 +299,7 @@ function projectParsed(value) {
// and provider-specific tool records are not inputs to reviewer artifacts.
function canonicalToolObservability(metadata) {
const source = record(record(record(metadata).agents_api).tool_observability)
if (source.version !== 1 || !Array.isArray(source.calls) || source.calls.length > MAX_TRANSCRIPT_EXECUTIONS) return undefined
if (source.version !== 1 || !Array.isArray(source.calls) || source.calls.length > MAX_TOOL_OBSERVABILITY_CALLS) return undefined
const calls = source.calls.map(projectCanonicalToolCall).filter(Boolean)
return calls.length ? { version: 1, calls } : undefined
}
Expand Down Expand Up @@ -354,7 +358,7 @@ async function trustedTranscriptFile(path) {
if (stat.isSymbolicLink()) throw new Error("Canonical transcript must not traverse symlinks.")
}
const stat = await lstat(current)
if (!stat.isFile() || stat.size > MAX_UPLOAD_FILE_BYTES) throw new Error("Canonical transcript must be a bounded regular file.")
if (!stat.isFile() || stat.size > MAX_TRANSCRIPT_SOURCE_BYTES) throw new Error("Canonical transcript must be a bounded regular file.")
const resolved = await realpath(current)
const contained = relative(root, resolved)
if (contained === ".." || contained.startsWith(`..${String.fromCharCode(47)}`) || isAbsolute(contained)) throw new Error("Canonical transcript escapes the trusted artifact root.")
Expand Down
4 changes: 3 additions & 1 deletion packages/runtime-core/src/tool-observability.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import { isPlainObject, objectValue, stripUndefined } from "./object-utils.js"

export const TOOL_OBSERVABILITY_VERSION = 1 as const
const MAX_TOOL_CALLS = 64
// Long agent runs (40+ turns with parallel tool calls) exceed 64 calls; the
// projection is content-free, so a larger bound stays cheap.
const MAX_TOOL_CALLS = 512
const MAX_IDENTIFIER_LENGTH = 256
const MAX_ARGUMENT_KEYS = 32
const RESULT_TYPES = ["array", "object", "string", "integer", "double", "boolean", "null"] as const
Expand Down
55 changes: 55 additions & 0 deletions tests/agent-runtime-failure.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import assert from "node:assert/strict"

import { agentRuntimeFailure } from "../.github/scripts/run-agent-task/agent-runtime-failure.mjs"

// A successful runtime has no failure.
assert.equal(agentRuntimeFailure({ execution: { code: 0 }, runtimeResult: { success: true } }), undefined)

// An unreadable native result (exit 0) names the reader diagnostic.
{
const failure = agentRuntimeFailure({
execution: { code: 0, stdout: "", stderr: "" },
runtimeResult: { success: false, diagnostics: [{ code: "wp-codebox.agent-task.result-too-large", message: "Native agent-task result is 2000000 bytes, exceeding the 1048576-byte limit." }] },
})
assert.equal(failure?.stage, "agent")
assert.equal(failure?.code, "wp-codebox.agent-task.result-too-large")
assert.match(String(failure?.message), /2000000 bytes/)
assert.equal(failure?.exit_code, 0)
assert.equal(failure?.stderr_tail, undefined, "stream tails are only attached to non-zero exits")
}

// A non-zero exit without diagnostics still yields a code and message with the stderr tail.
{
const failure = agentRuntimeFailure({ execution: { code: 1, stdout: "failed: task", stderr: "PHP Fatal error: boom" }, runtimeResult: {} })
assert.equal(failure?.code, "agent_runtime_exit")
assert.match(String(failure?.message), /exited with code 1.*PHP Fatal error: boom/)
assert.equal(failure?.stderr_tail, "PHP Fatal error: boom")
}

// Error diagnostics lead over warnings; provider errors classify the failure.
{
const failure = agentRuntimeFailure({
execution: { code: 1 },
runtimeResult: {
success: false,
agent_task_run_result: {
status: "provider_error",
diagnostics: [{ code: "slow", message: "slow", severity: "warning" }, { code: "provider_http_429", message: "Too many requests", severity: "error" }],
metadata: { provider_error: { message: "Too many requests", code: "429" } },
},
},
})
assert.equal(failure?.code, "provider_http_429")
assert.equal(failure?.classification, "provider_error")
assert.equal(failure?.runtime_status, "provider_error")
assert.deepEqual(failure?.provider_error, { message: "Too many requests", code: "429" })
}

// A failed terminal status without diagnostics becomes the code.
{
const failure = agentRuntimeFailure({ execution: { code: 0 }, runtimeResult: { success: false, terminal_result: { status: "timeout", failure_classification: "timeout" } } })
assert.equal(failure?.code, "agent_timeout")
assert.equal(failure?.classification, "timeout")
}

console.log("agent runtime failure ok")
Loading
Loading