From 12bb36f5329ed4933bd840545664c0e17f53a52b Mon Sep 17 00:00:00 2001 From: Will Eastler <32074613+weastler@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:20:17 -0800 Subject: [PATCH 1/4] feat(hub): configure repository build workflows Read build workflow selection from the repository default branch for webhook imports and recovery, preserving legacy defaults and immutable Builds. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../discover-hub/index.test.mjs | 3 +- .../scripts/validate-templates/index.mjs | 33 ++- .../scripts/validate-templates/index.test.mjs | 14 + hub/docs/github-actions-builds.md | 33 ++- hub/server/domains/github-builds/constants.ts | 1 - .../index.db.test.ts | 146 ++++++++- .../import-github-workflow-run/index.ts | 12 +- .../import-github-workflow-run/types.ts | 2 +- .../index.ts | 4 +- hub/server/index.ts | 7 + .../index.test.ts | 276 +++++++++++++++++- .../create-github-actions-client/index.ts | 134 +++++++-- .../find-connected-github-repository/index.ts | 11 + .../index.db.test.ts | 150 ++++++++++ .../index.ts | 15 +- .../read-build-workflow-paths/index.test.ts | 74 +++++ .../utils/read-build-workflow-paths/index.ts | 32 ++ hub/server/workers/github-recovery.test.ts | 70 +++++ 18 files changed, 978 insertions(+), 39 deletions(-) create mode 100644 hub/server/repositories/repository-access/find-connected-github-repository/index.ts create mode 100644 hub/server/utils/read-build-workflow-paths/index.test.ts create mode 100644 hub/server/utils/read-build-workflow-paths/index.ts diff --git a/.github/extensions/replica-prototype/discover-hub/index.test.mjs b/.github/extensions/replica-prototype/discover-hub/index.test.mjs index 237ff7a72..7174e8cf7 100644 --- a/.github/extensions/replica-prototype/discover-hub/index.test.mjs +++ b/.github/extensions/replica-prototype/discover-hub/index.test.mjs @@ -153,7 +153,7 @@ if (process.argv.some((arg) => arg.startsWith("expression="))) { } })) - it("validates only the Hub origin without rejecting the template catalog or unrelated schema fields", () => + it("validates only the Hub origin alongside build selection, templates and unrelated fields", () => fixture(async ({ context, options, state }) => { const templates = Array.from({ length: 100 }, (_, index) => ({ id: `template-${index}`, @@ -165,6 +165,7 @@ if (process.argv.some((arg) => arg.startsWith("expression="))) { state.text = JSON.stringify({ version, hub: { origin: "https://team.example.com/", additionalSetting: true }, + builds: { workflows: [".github/workflows/publish-proto.yml"] }, templates, anotherSetting: true, }) diff --git a/actions/build/scripts/validate-templates/index.mjs b/actions/build/scripts/validate-templates/index.mjs index b1b311e48..9b44ed2d9 100644 --- a/actions/build/scripts/validate-templates/index.mjs +++ b/actions/build/scripts/validate-templates/index.mjs @@ -16,11 +16,42 @@ export function validateTemplates(value) { typeof value !== "object" || Array.isArray(value) || value.version !== 1 || - Object.keys(value).some((key) => !["version", "hub", "templates"].includes(key)) + Object.keys(value).some((key) => !["version", "hub", "templates", "builds"].includes(key)) ) { throw new Error("Replica configuration must be a version 1 JSON object.") } + if (Object.hasOwn(value, "builds")) { + const builds = value.builds + if ( + !builds || + typeof builds !== "object" || + Array.isArray(builds) || + Object.keys(builds).some((key) => key !== "workflows") + ) { + throw new Error("Replica builds must be an object containing only workflows.") + } + if (Object.hasOwn(builds, "workflows")) { + const workflows = builds.workflows + if ( + !Array.isArray(workflows) || + workflows.length < 1 || + workflows.length > 10 || + workflows.some( + (workflow) => + typeof workflow !== "string" || + workflow.length > 255 || + !/^\.github\/workflows\/[A-Za-z0-9][A-Za-z0-9_.-]*\.ya?ml$/.test(workflow), + ) || + new Set(workflows).size !== workflows.length + ) { + throw new Error( + "Replica builds.workflows must contain 1 to 10 unique .github/workflows/.yml or .yaml paths (at most 255 characters each).", + ) + } + } + } + if (!Object.hasOwn(value, "templates")) return null if (!Array.isArray(value.templates) || value.templates.length > MAX_TEMPLATES) { diff --git a/actions/build/scripts/validate-templates/index.test.mjs b/actions/build/scripts/validate-templates/index.test.mjs index b0652b93b..7a6f98d21 100644 --- a/actions/build/scripts/validate-templates/index.test.mjs +++ b/actions/build/scripts/validate-templates/index.test.mjs @@ -12,6 +12,20 @@ const template = { } describe("validateTemplates()", () => { + it("allows build selection beside Hub and templates without packaging either setting", () => { + const metadata = { + version: 1, + hub: { origin: "https://replica.example.com" }, + builds: { workflows: [".github/workflows/publish-proto.yml"] }, + } + + assert.deepEqual(validateTemplates({ ...metadata, templates: [template] }), { + version: 1, + templates: [template], + }) + assert.equal(validateTemplates(metadata), null) + }) + it("preserves local paths, query strings and fragments but removes Hub metadata", () => { assert.deepEqual( validateTemplates({ diff --git a/hub/docs/github-actions-builds.md b/hub/docs/github-actions-builds.md index 7bc8de2a3..52c9a8254 100644 --- a/hub/docs/github-actions-builds.md +++ b/hub/docs/github-actions-builds.md @@ -1,15 +1,38 @@ # GitHub Actions Builds -Hub imports Builds that prototype repositories produce in `.github/workflows/replica.yml`. The workflow uploads static output with the [publish action](../../actions/build/publish/action.yml) and captures screenshots in a dependent job with the [screenshots action](../../actions/build/screenshots/action.yml). Hub pulls each package through the GitHub API. It imports the dist when the publish job finishes, without waiting for screenshots. A failed or missing screenshot job does not fail an already imported Build. +Hub imports Builds from selected GitHub Actions workflows in connected prototype repositories. The default is `.github/workflows/replica.yml`. The workflow uploads static output with the [publish action](../../actions/build/publish/action.yml) and captures screenshots in a dependent job with the [screenshots action](../../actions/build/screenshots/action.yml). Hub pulls each package through the GitHub API. It imports the dist when the publish job finishes, without waiting for screenshots. A failed or missing screenshot job does not fail an already imported Build. ## Requirements -The Hub that imports a repository's Builds is the Hub whose GitHub App is installed on that repository. The App needs **Actions: read** and **Contents: read** and must subscribe to the **Workflow runs**, **Workflow jobs**, **Push**, and **Branch or tag deletion** webhook events. Hub imports `push` runs of `.github/workflows/replica.yml` in connected repositories. To exclude tag runs, the workflow must restrict its trigger to branches. The installer branch is excluded. Imports never create repositories. +The Hub that imports a repository's Builds is the Hub whose GitHub App is installed on that repository. The App needs **Actions: read** and **Contents: read** and must subscribe to the **Workflow runs**, **Workflow jobs**, **Push**, and **Branch or tag deletion** webhook events. Hub imports only `push` runs of selected workflows in connected repositories, not pull request, `workflow_dispatch`, or `workflow_run` triggered runs. To exclude tag runs, each selected workflow must restrict its trigger to branches. The installer branch is excluded. Imports never create repositories. Create the App manually and connect it with the `setup:post` wizard. If the App was created before these settings were required, open its **Permissions & events** settings, confirm **Actions: read** and **Contents: read**, select these four events under **Subscribe to events**, and save. If either permission was added, each installation must accept the updated permissions. Webhooks give the fastest import. Without webhook reachability, [recovery](#recovery) still imports retained artifacts by polling, typically within about five minutes after Hub is available. Workflows do not contact Hub directly or need publishing credentials. +## Select build workflows + +Set `builds.workflows` in `.github/replica.json` on the connected repository's actual default branch. For example, Mini-Foundry can select its existing top-level caller: + +```json +{ + "version": 1, + "builds": { + "workflows": [".github/workflows/publish-proto.yml"] + } +} +``` + +Keep existing `hub` and `templates` fields alongside `builds`. The build action validates the new field but excludes it, like `hub`, from packaged template metadata. + +Hub reads the default branch from GitHub, resolves its commit, and reads configuration at that exact revision. It never uses the prototype branch's configuration, so existing prototype branches need no configuration update. Webhook intake, import workers, and recovery use the same validated selection. Each event, import job, or repository poll reads a fresh selection; changing the file affects subsequent work without replacing or deleting successful Builds. + +An omitted file, `builds`, or `builds.workflows` keeps the legacy default. An explicit list replaces that default; include `.github/workflows/replica.yml` explicitly to select it too. Use one to ten unique paths directly under `.github/workflows/`, ending in `.yml` or `.yaml`. Filenames must start with an ASCII letter or digit and contain only letters, digits, dots, underscores, or hyphens; each complete path is limited to 255 characters. Configuration must be a regular JSON file of at most 16 KiB. Invalid configuration stops imports and discovery with an error naming the configuration file; provider failures do not select a fallback workflow. + +GitHub identifies reusable workflow runs by the top-level caller, not the called workflow. Select `publish-proto.yml`, not Mini-Foundry's reusable `publish-proto-core.yml`. Selection alone does not add publishing jobs or make Azure output suitable for Hub: the selected caller must produce the existing Replica artifacts with a Hub-compatible root base path (`VITE_BASE=/`). This configuration does not modify downstream workflows or Azure publishing. + +When several selected workflows publish the same branch and commit, they share one Build. Existing run and attempt ordering still governs unfinished Builds. The first successful dist import remains immutable, and later screenshots attach only when their digest matches that dist. Select workflows that intentionally produce the same Hub output rather than competing variants. + ## Configure the prototype workflow Save this as `.github/workflows/replica.yml` in a connected prototype repository. Replace the build steps with the repository's own build; they must write static output, with an `index.html`, to `dist` or to the directory set in `output-directory`. @@ -83,13 +106,13 @@ sequenceDiagram Q->>I: download replica-screenshots-v1 → attach if digest matches ``` -The webhook accepts `workflow_run` (`requested`, `in_progress`, `completed`) for the Replica workflow path, and every completed `workflow_job`, because job events are the earliest signal that the dist artifact exists. `workflow_run.completed` arrives only after every job, including screenshots, so it cannot provide early preview availability. +The webhook accepts `workflow_run` (`requested`, `in_progress`, `completed`) for selected workflow paths, and every completed `workflow_job`, because job events are the earliest signal that the dist artifact exists. The import worker checks the selected caller path and push event again using GitHub's run record. `workflow_run.completed` arrives only after every job, including screenshots, so it cannot provide early preview availability. The route enqueues before recording the delivery receipt. A queue failure therefore leaves the delivery unrecorded, so [recovery](#recovery) can redeliver it. Each delivery gets its own job ID, so an event that arrives while an earlier import of the same run is active still triggers another pass. ## Import worker -`import-github-workflow-run` runs on the on-demand queue with five attempts and exponential backoff from 15 seconds. It reads the run through the GitHub API rather than trusting the webhook payload, and ignores runs that are not `push` runs of the Replica workflow, the installer branch, or unconnected repositories. +`import-github-workflow-run` runs on the on-demand queue with five attempts and exponential backoff from 15 seconds. It reads the run through the GitHub API rather than trusting the webhook payload, and ignores runs outside the selected workflows, non-`push` runs, the installer branch, and unconnected repositories. One Build exists per prototype identity and commit. Each pass is idempotent: @@ -153,7 +176,7 @@ Discovery and retry look back three days, matching the artifacts' retention. Run Branch polling selects a matching existing Build when a branch returns to an earlier commit; it does not invent a Build without a workflow package or delete branches. Poll observations do not replace workflow creation times or rewrite an unchanged prototype selection. Reselection retains the latest workflow timestamp and run ID so a known run's redelivery cannot undo it, while a new run can still become current. When an older installation has a selection timestamp but no saved run ID, runs with the same timestamp do not replace the selection because their order is unknown; a later timestamp can still select a Build. -Each poll bounds branch listing to ten pages of 100. Workflow discovery pages through recent runs within the recovery window, up to GitHub's 1,000-result filtered-search limit; reaching that limit reports an error instead of silently accepting a truncated list. Saved unfinished Builds are retried separately. A very busy repository can therefore outrun discovery of previously unseen runs; recovery is a safety net, not an unlimited delivery guarantee. Runs that reuse an immutable Build or have lost ownership to a newer run do not keep queuing redundant imports. Webhook imports and polling share immutable Build ownership, so whichever valid dist import commits first wins. +Each poll bounds branch listing to ten pages of 100. Workflow discovery queries each selected workflow and removes duplicate run IDs. It pages through recent runs within the recovery window, up to GitHub's 1,000-result filtered-search limit per workflow; reaching that limit reports an error instead of silently accepting a truncated list. A provider failure for any selected workflow rejects that repository's discovery pass rather than returning a partial list. Saved unfinished Builds are retried separately. A very busy repository can therefore outrun discovery of previously unseen runs; recovery is a safety net, not an unlimited delivery guarantee. Runs that reuse an immutable Build or have lost ownership to a newer run do not keep queuing redundant imports. Webhook imports and polling share immutable Build ownership, so whichever valid dist import commits first wins. ## Not yet implemented diff --git a/hub/server/domains/github-builds/constants.ts b/hub/server/domains/github-builds/constants.ts index ce57a63ee..a5ddfde12 100644 --- a/hub/server/domains/github-builds/constants.ts +++ b/hub/server/domains/github-builds/constants.ts @@ -1,6 +1,5 @@ export const distArtifactName = "replica-build-v1" export const screenshotArtifactName = "replica-screenshots-v1" -export const replicaWorkflowPath = ".github/workflows/replica.yml" // Leaves room for ZIP headers around a package at the extracted-size limit. export const maximumArtifactDownloadBytes = 160 * 1024 * 1024 // Matches the 3-day retention of the Replica Build artifacts; older ones have expired. diff --git a/hub/server/domains/github-builds/import-github-workflow-run/index.db.test.ts b/hub/server/domains/github-builds/import-github-workflow-run/index.db.test.ts index 49dac6955..cbb447210 100644 --- a/hub/server/domains/github-builds/import-github-workflow-run/index.db.test.ts +++ b/hub/server/domains/github-builds/import-github-workflow-run/index.db.test.ts @@ -81,8 +81,10 @@ async function createFixture() { actor: { id: 9, login: "octocat", avatarUrl: null }, } const packages = new Map() + const workflowPaths = [".github/workflows/replica.yml"] const hooks: { beforeDownload?: (id: number) => Promise } = {} const client = { + getBuildWorkflowPaths: () => Promise.resolve([...workflowPaths]), getWorkflowRun: () => Promise.resolve({ ...run }), listRunArtifacts: (_runId: number, name: string) => Promise.resolve( @@ -122,10 +124,152 @@ async function createFixture() { orderBy: { id: "asc" }, }) } - return { run, packages, hooks, importRun, findBuild, findAnalyticsEvents, storage, client } + return { + run, + packages, + hooks, + importRun, + findBuild, + findAnalyticsEvents, + storage, + client, + workflowPaths, + } } describe("importGitHubWorkflowRun", () => { + it("imports a selected custom caller from its webhook without letting a competing selected workflow replace its dist", async () => { + const fixture = await createFixture() + fixture.workflowPaths.splice( + 0, + 1, + ".github/workflows/publish-proto.yml", + ".github/workflows/preview.yaml", + ) + fixture.run.path = ".github/workflows/publish-proto.yml@refs/heads/proto/custom" + fixture.run.headBranch = "proto/custom" + const originalDist = distPackage("

Selected caller

") + const originalDigest = extractDistPackage(originalDist).distDigest + fixture.packages.set(distArtifactName, originalDist) + const enqueued: GitHubWorkflowRunJob[] = [] + const webhooks = createGitHubWebhookRouteDependencies({ + installationId: "700", + provider: { configurationDigest: "test-provider", revision: 1 }, + webhookSecret: "webhook-secret-for-tests", + syncRepositories: () => Promise.resolve(), + readBuildWorkflowPaths: () => fixture.client.getBuildWorkflowPaths(), + enqueueWorkflowRunImport: (job) => { + enqueued.push(job) + return Promise.resolve() + }, + }) + const payload = { + action: "in_progress", + repository: { id: fixture.run.repositoryId }, + workflow_run: { id: fixture.run.id, path: fixture.run.path }, + } + + await expect( + webhooks.handleEvent("workflow_run", payload, { + id: randomUUID(), + payloadDigest: createHash("sha256").update(JSON.stringify(payload)).digest("hex"), + }), + ).resolves.toBe(true) + expect(enqueued).toHaveLength(1) + await expect(fixture.importRun(enqueued[0])).resolves.toEqual({ + dist: "imported", + screenshots: "absent", + }) + const original = await fixture.findBuild() + expect(original.prototype.registeredAt).toBeInstanceOf(Date) + + fixture.run.id += 1 + fixture.run.path = ".github/workflows/preview.yaml" + fixture.run.createdAt = new Date("2026-01-01T00:02:00Z") + const competingDist = distPackage("

Competing caller

") + fixture.packages.set(distArtifactName, competingDist) + fixture.packages.set( + screenshotArtifactName, + screenshotPackage(extractDistPackage(competingDist).distDigest), + ) + + await expect(fixture.importRun()).resolves.toEqual({ + dist: "unchanged", + screenshots: "digest-mismatch", + }) + expect(await fixture.findBuild()).toMatchObject({ + id: original.id, + githubWorkflowRunId: original.githubWorkflowRunId, + distDigest: originalDigest, + artifacts: original.artifacts, + state: "successful", + }) + const preview = await readBuildPreviewFile( + { acceptsHtml: true, buildId: original.id, requestPath: "/" }, + { + storage: fixture.storage, + findArtifact: (buildId, file) => findBuildArtifact(prisma, buildId, file), + }, + ) + expect(preview.status).toBe("ok") + if (preview.status === "ok") { + expect(Buffer.from(preview.bytes).toString()).toContain("

Selected caller

") + expect(Buffer.from(preview.bytes).toString()).not.toContain("Competing caller") + } + + fixture.packages.set(screenshotArtifactName, screenshotPackage(originalDigest)) + await expect(fixture.importRun()).resolves.toEqual({ + dist: "unchanged", + screenshots: "attached", + }) + expect((await fixture.findBuild()).artifacts.map((artifact) => artifact.path)).toContain( + "screenshots/desktop.png", + ) + }) + + it("does not fetch workflow configuration or runs for a disconnected repository", async () => { + const fixture = await createFixture() + await prisma.repository.update({ + where: { githubRepositoryId: fixture.run.repositoryId }, + data: { removedAt: new Date() }, + }) + const selection = vi.spyOn(fixture.client, "getBuildWorkflowPaths") + const getRun = vi.spyOn(fixture.client, "getWorkflowRun") + + await expect(fixture.importRun()).resolves.toEqual({ dist: "ignored", screenshots: "absent" }) + expect(selection).not.toHaveBeenCalled() + expect(getRun).not.toHaveBeenCalled() + }) + + it("propagates configuration failures before reading or preparing a workflow run", async () => { + const fixture = await createFixture() + vi.spyOn(fixture.client, "getBuildWorkflowPaths").mockRejectedValue( + new Error(".github/replica.json builds.workflows is invalid."), + ) + const getRun = vi.spyOn(fixture.client, "getWorkflowRun") + + await expect(fixture.importRun()).rejects.toThrow("builds.workflows") + expect(getRun).not.toHaveBeenCalled() + expect(await prisma.build.count()).toBe(0) + }) + + it.each([ + { path: ".github/workflows/replica.yml" }, + { path: ".github/workflows/publish-proto.yml-extra" }, + { event: "pull_request" }, + { headBranch: null }, + { headBranch: "replica/install-workflow" }, + { repositoryId: 0 }, + { headSha: "not-an-immutable-commit" }, + ])("keeps existing run safety checks for custom workflow selection: %j", async (changes) => { + const fixture = await createFixture() + fixture.workflowPaths.splice(0, 1, ".github/workflows/publish-proto.yml") + Object.assign(fixture.run, { path: ".github/workflows/publish-proto.yml" }, changes) + + await expect(fixture.importRun()).resolves.toEqual({ dist: "ignored", screenshots: "absent" }) + expect(await prisma.build.count()).toBe(0) + }) + it("serves the dist preview after a completed job while the workflow is still running", async () => { const fixture = await createFixture() fixture.run.headBranch = "proto/welcome/nested" diff --git a/hub/server/domains/github-builds/import-github-workflow-run/index.ts b/hub/server/domains/github-builds/import-github-workflow-run/index.ts index ceeacd22d..dfd01ae71 100644 --- a/hub/server/domains/github-builds/import-github-workflow-run/index.ts +++ b/hub/server/domains/github-builds/import-github-workflow-run/index.ts @@ -1,6 +1,7 @@ import { completeGitHubBuildFailure } from "../../../repositories/github-builds/complete-github-build-failure/index.ts" import { markGitHubBuildRunning } from "../../../repositories/github-builds/mark-github-build-running/index.ts" import { recordGitHubBuildRunCompletion } from "../../../repositories/github-builds/record-github-build-run-completion/index.ts" +import { findConnectedGitHubRepository } from "../../../repositories/repository-access/find-connected-github-repository/index.ts" import { BuildPackageError } from "../../build-artifacts/errors.ts" import { extractScreenshotPackage } from "../../build-artifacts/extract-screenshot-package/index.ts" import { registerImportedPrototype } from "../../prototypes/register-imported-prototype/index.ts" @@ -41,10 +42,19 @@ export async function importGitHubWorkflowRun( job: GitHubWorkflowRunJob, ): Promise { const { client, database, logger } = dependencies + if (!(await findConnectedGitHubRepository(database, job.githubRepositoryId))) { + return { dist: "ignored", screenshots: "absent" } + } + const workflowPaths = await client.getBuildWorkflowPaths() const found = await client.getWorkflowRun(job.runId) if (!found) return { dist: "ignored", screenshots: "absent" } const run = found - const prepared = await prepareGitHubWorkflowRunBuild(database, job.githubRepositoryId, run) + const prepared = await prepareGitHubWorkflowRunBuild( + database, + job.githubRepositoryId, + run, + workflowPaths, + ) if (prepared.status !== "ready") return { dist: "ignored", screenshots: "absent" } const { buildId, buildRun, distDigest, ownsBuild, state } = prepared const completed = run.status === "completed" diff --git a/hub/server/domains/github-builds/import-github-workflow-run/types.ts b/hub/server/domains/github-builds/import-github-workflow-run/types.ts index c7e8018ab..b1fb3c76a 100644 --- a/hub/server/domains/github-builds/import-github-workflow-run/types.ts +++ b/hub/server/domains/github-builds/import-github-workflow-run/types.ts @@ -4,7 +4,7 @@ import type { GitHubBuildImportDependencies } from "../types.ts" export interface ImportGitHubWorkflowRunDependencies extends GitHubBuildImportDependencies { client: Pick< ReturnType, - "getWorkflowRun" | "listRunArtifacts" | "downloadArtifact" + "getBuildWorkflowPaths" | "getWorkflowRun" | "listRunArtifacts" | "downloadArtifact" > } diff --git a/hub/server/domains/github-builds/prepare-github-workflow-run-build/index.ts b/hub/server/domains/github-builds/prepare-github-workflow-run-build/index.ts index 78c9316c3..86d9a113e 100644 --- a/hub/server/domains/github-builds/prepare-github-workflow-run-build/index.ts +++ b/hub/server/domains/github-builds/prepare-github-workflow-run-build/index.ts @@ -1,7 +1,6 @@ import type { PrismaClient } from "../../../generated/prisma/client.ts" import type { GitHubWorkflowRun } from "../../../integrations/github/create-github-actions-client/types.ts" import { prepareGitHubBuild } from "../../../repositories/github-builds/prepare-github-build/index.ts" -import { replicaWorkflowPath } from "../constants.ts" /** * Finds or creates the Build for a Replica workflow branch push run, using GitHub's @@ -11,11 +10,12 @@ export async function prepareGitHubWorkflowRunBuild( database: PrismaClient, githubRepositoryId: number, run: GitHubWorkflowRun, + workflowPaths: string[], ) { const branchName = run.headBranch if ( run.repositoryId !== githubRepositoryId || - run.path.split("@")[0] !== replicaWorkflowPath || + !workflowPaths.includes(run.path.split("@")[0] ?? "") || run.event !== "push" || !branchName || branchName === "replica/install-workflow" || diff --git a/hub/server/index.ts b/hub/server/index.ts index ccae13415..f2e32b42f 100644 --- a/hub/server/index.ts +++ b/hub/server/index.ts @@ -218,6 +218,8 @@ try { const context = createGitHubAuthenticationContext(authenticationEnvironment) const { createGitHubWebhookRouteDependencies } = await import("./routes/api/github/webhooks/create-github-webhook-route-dependencies/index.ts") + const { createGitHubActionsClient } = + await import("./integrations/github/create-github-actions-client/index.ts") githubWebhooks = createGitHubWebhookRouteDependencies({ installationId: appCredentials.installationId, provider: { @@ -226,6 +228,11 @@ try { }, syncRepositories: () => setup.syncRepositories(), webhookSecret: appCredentials.webhookSecret, + readBuildWorkflowPaths: (githubRepositoryId) => + createGitHubActionsClient({ + ...appCredentials, + githubRepositoryId, + }).getBuildWorkflowPaths(), enqueueWorkflowRunImport: async (job, deliveryId) => { const { enqueueGitHubWorkflowRunImport } = await import("./domains/github-builds/enqueue-github-workflow-run-import/index.ts") diff --git a/hub/server/integrations/github/create-github-actions-client/index.test.ts b/hub/server/integrations/github/create-github-actions-client/index.test.ts index 77d274c99..556d33f55 100644 --- a/hub/server/integrations/github/create-github-actions-client/index.test.ts +++ b/hub/server/integrations/github/create-github-actions-client/index.test.ts @@ -8,16 +8,43 @@ const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }) const input = { appId: "12", installationId: "34", privateKey, githubRepositoryId: 56 } type Handler = (url: URL, init: RequestInit) => Response -function respond(handler: Handler) { +function respond( + handler: Handler, + configuration?: (url: URL, init: RequestInit) => Response | undefined, +) { return vi.fn((url: URL | RequestInfo, init?: RequestInit) => { const href = new URL(url instanceof Request ? url.url : url.toString()) if (href.pathname === "/app/installations/34/access_tokens") { return Promise.resolve(new Response(JSON.stringify({ token: "t" }), { status: 201 })) } + if ( + href.pathname === "/repositories/56" || + href.pathname.startsWith("/repositories/56/git/ref/heads/") || + href.pathname === "/repositories/56/contents/.github/replica.json" + ) { + const overridden = configuration?.(href, init ?? {}) + if (overridden) return Promise.resolve(overridden) + if (href.pathname === "/repositories/56") { + return Promise.resolve(Response.json({ id: 56, default_branch: "trunk" })) + } + if (href.pathname.includes("/git/ref/")) { + return Promise.resolve(Response.json({ object: { type: "commit", sha: "c".repeat(40) } })) + } + return Promise.resolve(new Response("{}", { status: 404 })) + } return Promise.resolve(handler(href, init ?? {})) }) } +function configurationFile(content: string) { + return Response.json({ + type: "file", + encoding: "base64", + size: Buffer.byteLength(content), + content: Buffer.from(content).toString("base64"), + }) +} + function workflowRun(id: number, createdAt = "2026-01-02T00:00:00Z") { return { id, @@ -37,6 +64,253 @@ function workflowRun(id: number, createdAt = "2026-01-02T00:00:00Z") { } describe("createGitHubActionsClient", () => { + it("pins selection to the actual default branch commit and caches it only for one client", async () => { + let filename = "publish-proto.yml" + const fetch = respond( + () => Response.json({ workflow_runs: [] }), + (url) => + url.pathname.endsWith("/replica.json") + ? configurationFile( + JSON.stringify({ + version: 1, + hub: { origin: "https://hub.example.test" }, + templates: [], + builds: { workflows: [`.github/workflows/${filename}`] }, + }), + ) + : undefined, + ) + const client = createGitHubActionsClient(input, { fetch }) + + await expect( + Promise.all([client.getBuildWorkflowPaths(), client.getBuildWorkflowPaths()]), + ).resolves.toEqual([ + [".github/workflows/publish-proto.yml"], + [".github/workflows/publish-proto.yml"], + ]) + filename = "next.yaml" + await client.listWorkflowRuns(new Date("2026-01-01T00:00:00Z")) + await expect(client.getBuildWorkflowPaths()).resolves.toEqual([ + ".github/workflows/publish-proto.yml", + ]) + + const requests = fetch.mock.calls.map(([url]) => url as URL) + expect(requests.filter((url) => url.pathname === "/repositories/56")).toHaveLength(1) + expect( + requests.filter((url) => url.pathname === "/repositories/56/git/ref/heads/trunk"), + ).toHaveLength(1) + const configReads = requests.filter((url) => url.pathname.endsWith("/replica.json")) + expect(configReads).toHaveLength(1) + expect(configReads[0]?.searchParams.get("ref")).toBe("c".repeat(40)) + expect( + requests + .filter((url) => url.pathname.includes("/actions/workflows/")) + .map((url) => url.pathname), + ).toEqual(["/repositories/56/actions/workflows/publish-proto.yml/runs"]) + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).resolves.toEqual([".github/workflows/next.yaml"]) + }) + + it("pages every selected workflow independently and deduplicates run IDs across them", async () => { + const workflows = [".github/workflows/publish-proto.yml", ".github/workflows/preview.yaml"] + const fetch = respond( + (url) => { + const path = url.pathname.includes("publish-proto.yml") ? workflows[0] : workflows[1] + const ids = path === workflows[0] ? [7, 6, 5] : [5, 4, 3] + const page = Number(url.searchParams.get("page")) + const runs = ids.slice((page - 1) * 2, page * 2).map((id) => ({ + ...workflowRun(id), + path, + })) + return Response.json({ total_count: 3, workflow_runs: runs }) + }, + (url) => + url.pathname.endsWith("/replica.json") + ? configurationFile(JSON.stringify({ version: 1, builds: { workflows } })) + : undefined, + ) + + const runs = await createGitHubActionsClient(input, { fetch }).listWorkflowRuns( + new Date("2026-01-01T00:00:00Z"), + 2, + ) + + expect(runs.map((run) => run.id).sort()).toEqual([3, 4, 5, 6, 7]) + const requests = fetch.mock.calls + .map(([url]) => url as URL) + .filter((url) => url.pathname.includes("/actions/workflows/")) + expect( + requests.map((url) => [url.pathname.split("/").at(-2), url.searchParams.get("page")]), + ).toEqual([ + ["publish-proto.yml", "1"], + ["publish-proto.yml", "2"], + ["preview.yaml", "1"], + ["preview.yaml", "2"], + ]) + expect(requests.every((url) => url.searchParams.get("event") === "push")).toBe(true) + }) + + it.each(["provider error", "truncated results"])( + "does not return earlier workflow results after a later workflow has %s", + async (failure) => { + const workflows = [".github/workflows/first.yml", ".github/workflows/second.yaml"] + const fetch = respond( + (url) => { + if (url.pathname.includes("/first.yml/")) { + return Response.json({ workflow_runs: [workflowRun(7)] }) + } + if (failure === "provider error") return new Response("{}", { status: 503 }) + const size = Number(url.searchParams.get("per_page")) + return Response.json({ + total_count: 1_001, + workflow_runs: Array.from({ length: size }, (_, index) => workflowRun(100 + index)), + }) + }, + (url) => + url.pathname.endsWith("/replica.json") + ? configurationFile(JSON.stringify({ version: 1, builds: { workflows } })) + : undefined, + ) + + await expect( + createGitHubActionsClient(input, { fetch }).listWorkflowRuns( + new Date("2026-01-01T00:00:00Z"), + ), + ).rejects.toThrow(GitHubProviderRequestError) + }, + ) + + it("continues discovery when one selected workflow has not produced any runs", async () => { + const fetch = respond( + (url) => + url.pathname.includes("/missing.yml/") + ? new Response("{}", { status: 404 }) + : Response.json({ + workflow_runs: [{ ...workflowRun(7), path: ".github/workflows/publish.yaml" }], + }), + (url) => + url.pathname.endsWith("/replica.json") + ? configurationFile( + JSON.stringify({ + version: 1, + builds: { + workflows: [".github/workflows/missing.yml", ".github/workflows/publish.yaml"], + }, + }), + ) + : undefined, + ) + + await expect( + createGitHubActionsClient(input, { fetch }).listWorkflowRuns( + new Date("2026-01-01T00:00:00Z"), + ), + ).resolves.toEqual([expect.objectContaining({ id: 7 })]) + }) + + it.each([401, 403, 429, 503])( + "does not treat a configuration HTTP %s failure as omitted configuration", + async (status) => { + const fetch = respond( + () => Response.json({ workflow_runs: [] }), + (url) => + url.pathname.endsWith("/replica.json") ? new Response("{}", { status }) : undefined, + ) + + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).rejects.toThrow(GitHubProviderRequestError) + }, + ) + + it.each(["/repositories/56", "/repositories/56/git/ref/heads/trunk"])( + "does not treat missing source evidence at %s as omitted configuration", + async (path) => { + const fetch = respond( + () => Response.json({}), + (url) => (url.pathname === path ? new Response("{}", { status: 404 }) : undefined), + ) + + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).rejects.toThrow(GitHubProviderRequestError) + }, + ) + + it("propagates transport failures rather than falling back to the legacy workflow", async () => { + const fetch = respond( + () => Response.json({}), + (url) => { + if (url.pathname.endsWith("/replica.json")) throw new TypeError("Connection reset") + return undefined + }, + ) + + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).rejects.toThrow(GitHubProviderRequestError) + }) + + it.each([ + { type: "symlink", encoding: "base64", size: 13, content: "eyJ2ZXJzaW9uIjoxfQ==" }, + { type: "file", encoding: "utf-8", size: 13, content: '{"version":1}' }, + { type: "file", encoding: "base64", size: 13, content: "not base64 JSON" }, + ])( + "rejects malformed configuration file evidence instead of using a fallback: %j", + async (file) => { + const fetch = respond( + () => Response.json({}), + (url) => (url.pathname.endsWith("/replica.json") ? Response.json(file) : undefined), + ) + + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).rejects.toThrow(/replica\.json|configuration/i) + }, + ) + + it.each(["{invalid", '{"version":1,"builds":{"workflows":[]}}'])( + "reports malformed explicit configuration instead of selecting the legacy workflow: %s", + async (content) => { + const fetch = respond( + () => Response.json({}), + (url) => (url.pathname.endsWith("/replica.json") ? configurationFile(content) : undefined), + ) + + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).rejects.toThrow(/replica\.json|configuration|workflows/i) + }, + ) + + it("bounds decoded configuration bytes rather than the encoded response length", async () => { + const content = '{"version":1}'.padEnd(16_384, " ") + const fetch = respond( + () => Response.json({}), + (url) => (url.pathname.endsWith("/replica.json") ? configurationFile(content) : undefined), + ) + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).resolves.toEqual([".github/workflows/replica.yml"]) + + const oversized = respond( + () => Response.json({}), + (url) => + url.pathname.endsWith("/replica.json") + ? Response.json({ + type: "file", + encoding: "base64", + size: 1, + content: Buffer.from(`${content} `).toString("base64"), + }) + : undefined, + ) + await expect( + createGitHubActionsClient(input, { fetch: oversized }).getBuildWorkflowPaths(), + ).rejects.toThrow(/16384|16 KiB|large|size|bytes/i) + }) + it("requests one repository-scoped token for Actions and branch reads", async () => { const fetch = respond(() => new Response(JSON.stringify({ total_count: 0, artifacts: [] }))) const client = createGitHubActionsClient(input, { fetch }) diff --git a/hub/server/integrations/github/create-github-actions-client/index.ts b/hub/server/integrations/github/create-github-actions-client/index.ts index 8f2b04cf6..1a5966080 100644 --- a/hub/server/integrations/github/create-github-actions-client/index.ts +++ b/hub/server/integrations/github/create-github-actions-client/index.ts @@ -1,5 +1,6 @@ import { z } from "zod" +import { readBuildWorkflowPaths } from "../../../utils/read-build-workflow-paths/index.ts" import { createGitHubAppJwt } from "../create-github-app-jwt/index.ts" import { GitHubProviderRequestError } from "../errors.ts" import type { @@ -59,6 +60,23 @@ const artifactsSchema = z.looseObject({ ), }) const maximumArtifactPages = 10 +const maximumConfigurationBytes = 16_384 +const repositorySchema = z.looseObject({ + id: z.number().int().positive(), + default_branch: z.string().min(1).max(255), +}) +const referenceSchema = z.looseObject({ + object: z.looseObject({ + type: z.literal("commit"), + sha: z.string().regex(/^[0-9a-f]{40}$/), + }), +}) +const configurationFileSchema = z.looseObject({ + type: z.literal("file"), + encoding: z.literal("base64"), + size: z.number().int().nonnegative().max(maximumConfigurationBytes), + content: z.string(), +}) function mapRun(run: z.infer): GitHubWorkflowRun { const actor = run.triggering_actor ?? run.actor @@ -115,6 +133,7 @@ export function createGitHubActionsClient( }: GitHubActionsClientOptions = {}, ) { let token: Promise | undefined + let workflowPaths: Promise | undefined const repositoryPath = `/repositories/${input.githubRepositoryId}` async function send(path: string, authorization: string, init: RequestInit = {}) { @@ -164,7 +183,60 @@ export function createGitHubActionsClient( return token } + async function getBuildWorkflowPaths(): Promise { + workflowPaths ??= (async () => { + const authorization = await installationToken() + const repository = await parse(repositorySchema, await send(repositoryPath, authorization)) + if (repository.id !== input.githubRepositoryId) throw new GitHubProviderRequestError() + const reference = await parse( + referenceSchema, + await send( + `${repositoryPath}/git/ref/heads/${encodeURIComponent(repository.default_branch)}`, + authorization, + ), + ) + const response = await send( + `${repositoryPath}/contents/.github/replica.json?ref=${reference.object.sha}`, + authorization, + ) + if (response.status === 404) return readBuildWorkflowPaths({ version: 1 }) + if (!response.ok) throw new GitHubProviderRequestError() + const bytes = await readBounded(response, 32_768) + if (!bytes) { + throw new Error(".github/replica.json exceeds the 16,384-byte configuration limit.") + } + let value: unknown + try { + value = JSON.parse(bytes.toString("utf8")) + } catch (error) { + throw new GitHubProviderRequestError({ cause: error }) + } + const file = configurationFileSchema.safeParse(value) + if (!file.success) { + throw new Error( + ".github/replica.json must be a regular JSON file of at most 16,384 bytes on the default branch.", + { cause: file.error }, + ) + } + const content = Buffer.from(file.data.content, "base64") + if (content.length > maximumConfigurationBytes || content.length !== file.data.size) { + throw new Error(".github/replica.json has an invalid or oversized file size.") + } + let configuration: unknown + try { + configuration = JSON.parse(content.toString("utf8")) + } catch (error) { + throw new Error(".github/replica.json must contain correctly formatted JSON.", { + cause: error, + }) + } + return readBuildWorkflowPaths(configuration) + })() + return workflowPaths + } + return { + getBuildWorkflowPaths, /** Lists branch heads without assuming a truncated response is a complete snapshot. */ async listBranchHeads(): Promise<{ branchName: string; commitSha: string }[]> { const heads: { branchName: string; commitSha: string }[] = [] @@ -222,32 +294,54 @@ export function createGitHubActionsClient( }, /** - * Lists push runs of the Replica workflow through createdAfter. GitHub caps filtered + * Lists push runs of selected workflows through createdAfter. GitHub caps filtered * run searches at 1,000 results; report truncation instead of silently missing runs. */ async listWorkflowRuns(createdAfter: Date, perPage = 50): Promise { + if (!Number.isInteger(perPage) || perPage < 1 || perPage > 100) { + throw new Error("GitHub workflow discovery page size must be between 1 and 100.") + } const runs: GitHubWorkflowRun[] = [] - for (let page = 1; page <= Math.ceil(1_000 / perPage); page += 1) { - const query = new URLSearchParams({ - event: "push", - created: `>=${createdAfter.toISOString().replace(/\.\d{3}Z$/, "Z")}`, - per_page: String(perPage), - page: String(page), - }) - const response = await send( - `${repositoryPath}/actions/workflows/replica.yml/runs?${query}`, - await installationToken(), - ) - if (response.status === 404 && page === 1) return [] - const found = (await parse(runsSchema, response)).workflow_runs.map(mapRun) - runs.push(...found.filter((run) => run.createdAt >= createdAfter)) - if (found.length < perPage || found.some((run) => run.createdAt < createdAfter)) { - return runs + const paths = await getBuildWorkflowPaths() + for (const path of paths) { + let complete = false + for (let page = 1; page <= Math.ceil(1_000 / perPage); page += 1) { + const query = new URLSearchParams({ + event: "push", + created: `>=${createdAfter.toISOString().replace(/\.\d{3}Z$/, "Z")}`, + per_page: String(perPage), + page: String(page), + }) + const response = await send( + `${repositoryPath}/actions/workflows/${encodeURIComponent(path.slice(".github/workflows/".length))}/runs?${query}`, + await installationToken(), + ) + if (response.status === 404 && page === 1) { + complete = true + break + } + const found = (await parse(runsSchema, response)).workflow_runs.map(mapRun) + runs.push( + ...found.filter( + (run) => + run.createdAt >= createdAfter && + run.repositoryId === input.githubRepositoryId && + run.event === "push" && + paths.includes(run.path.split("@")[0] ?? ""), + ), + ) + if (found.length < perPage || found.some((run) => run.createdAt < createdAfter)) { + complete = true + break + } + } + if (!complete) { + throw new GitHubProviderRequestError({ + cause: new Error("Workflow run discovery reached GitHub's 1,000-result search limit."), + }) } } - throw new GitHubProviderRequestError({ - cause: new Error("Workflow run discovery reached GitHub's 1,000-result search limit."), - }) + return [...new Map(runs.map((run) => [run.id, run])).values()] }, async listRunArtifacts(runId: number, name: string): Promise { diff --git a/hub/server/repositories/repository-access/find-connected-github-repository/index.ts b/hub/server/repositories/repository-access/find-connected-github-repository/index.ts new file mode 100644 index 000000000..279508cae --- /dev/null +++ b/hub/server/repositories/repository-access/find-connected-github-repository/index.ts @@ -0,0 +1,11 @@ +import type { Prisma } from "../../../generated/prisma/client.ts" + +export async function findConnectedGitHubRepository( + database: Pick, + githubRepositoryId: number, +) { + return database.repository.findFirst({ + where: { githubRepositoryId: BigInt(githubRepositoryId), removedAt: null }, + select: { id: true }, + }) +} diff --git a/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.db.test.ts b/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.db.test.ts index 3e254f790..0cf7da3fc 100644 --- a/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.db.test.ts +++ b/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.db.test.ts @@ -1048,9 +1048,11 @@ describe("createGitHubWebhookRouteDependencies()", () => { }) it("queues a Replica workflow import before recording the delivery", async () => { + await createRepository(1001, "workflows") const enqueueWorkflowRunImport = vi.fn(() => Promise.resolve()) const dependencies = createGitHubWebhookRouteDependencies({ enqueueWorkflowRunImport, + readBuildWorkflowPaths: () => Promise.resolve([".github/workflows/replica.yml"]), installationId: "700", provider: githubProvider, syncRepositories: () => Promise.resolve(), @@ -1093,8 +1095,10 @@ describe("createGitHubWebhookRouteDependencies()", () => { }) it("leaves a workflow delivery unrecorded when queueing fails so GitHub can redeliver it", async () => { + await createRepository(1001, "workflows") const dependencies = createGitHubWebhookRouteDependencies({ enqueueWorkflowRunImport: () => Promise.reject(new Error("Queue unavailable.")), + readBuildWorkflowPaths: () => Promise.resolve([".github/workflows/replica.yml"]), installationId: "700", provider: githubProvider, syncRepositories: () => Promise.resolve(), @@ -1113,6 +1117,152 @@ describe("createGitHubWebhookRouteDependencies()", () => { expect(await prisma.gitHubWebhookDelivery.count()).toBe(0) }) + it("queues only selected workflow callers while leaving completed jobs for the worker to verify", async () => { + await createRepository(1001, "custom-workflows") + const readBuildWorkflowPaths = vi.fn(() => + Promise.resolve([".github/workflows/publish-proto.yml", ".github/workflows/preview.yaml"]), + ) + const enqueueWorkflowRunImport = vi.fn(() => Promise.resolve()) + const dependencies = createGitHubWebhookRouteDependencies({ + installationId: "700", + provider: githubProvider, + syncRepositories: () => Promise.resolve(), + webhookSecret: "webhook-secret-for-tests", + readBuildWorkflowPaths, + enqueueWorkflowRunImport, + }) + for (const [index, path] of [ + ".github/workflows/publish-proto.yml@refs/heads/proto/custom", + ".github/workflows/preview.yaml", + ".github/workflows/replica.yml", + ".github/workflows/publish-proto.yml-extra", + ".github/workflows/nested/publish-proto.yml", + ].entries()) { + const payload = { + action: "completed", + repository: { id: 1001 }, + workflow_run: { id: 50 + index, path }, + } + await expect( + dependencies.handleEvent("workflow_run", payload, deliveryFor(payload)), + ).resolves.toBe(index < 2) + } + const readsBeforeJob = readBuildWorkflowPaths.mock.calls.length + const job = { + action: "completed", + repository: { id: 1001 }, + workflow_job: { run_id: 99 }, + } + await expect(dependencies.handleEvent("workflow_job", job, deliveryFor(job))).resolves.toBe( + true, + ) + + for (const [index, runId] of [50, 51, 99].entries()) { + expect(enqueueWorkflowRunImport).toHaveBeenNthCalledWith( + index + 1, + { githubRepositoryId: 1001, runId }, + expect.any(String), + ) + } + expect(readBuildWorkflowPaths).toHaveBeenCalledWith(1001) + expect(readBuildWorkflowPaths).toHaveBeenCalledTimes(readsBeforeJob) + expect(await prisma.gitHubWebhookDelivery.count()).toBe(3) + }) + + it.each(["unknown", "disconnected"])( + "does not read configuration for an %s repository's workflow", + async (state) => { + if (state === "disconnected") { + const repository = await createRepository(1001, "removed-workflows") + await prisma.repository.update({ + where: { id: repository.id }, + data: { removedAt: new Date() }, + }) + } + const readBuildWorkflowPaths = vi.fn(() => + Promise.reject(new Error("Must not contact an unconnected repository.")), + ) + const enqueueWorkflowRunImport = vi.fn(() => Promise.resolve()) + const dependencies = createGitHubWebhookRouteDependencies({ + installationId: "700", + provider: githubProvider, + syncRepositories: () => Promise.resolve(), + webhookSecret: "webhook-secret-for-tests", + readBuildWorkflowPaths, + enqueueWorkflowRunImport, + }) + const payload = { + action: "completed", + repository: { id: 1001 }, + workflow_run: { id: 55, path: ".github/workflows/replica.yml" }, + } + + await expect( + dependencies.handleEvent("workflow_run", payload, deliveryFor(payload)), + ).resolves.toBe(false) + expect(readBuildWorkflowPaths).not.toHaveBeenCalled() + expect(enqueueWorkflowRunImport).not.toHaveBeenCalled() + expect(await prisma.gitHubWebhookDelivery.count()).toBe(0) + }, + ) + + it("fails clearly when workflow selection is unwired instead of silently ignoring a valid run", async () => { + await createRepository(1001, "unwired-workflows") + const enqueueWorkflowRunImport = vi.fn(() => Promise.resolve()) + const dependencies = createGitHubWebhookRouteDependencies({ + installationId: "700", + provider: githubProvider, + syncRepositories: () => Promise.resolve(), + webhookSecret: "webhook-secret-for-tests", + enqueueWorkflowRunImport, + }) + const payload = { + action: "completed", + repository: { id: 1001 }, + workflow_run: { id: 55, path: ".github/workflows/replica.yml" }, + } + + await expect( + dependencies.handleEvent("workflow_run", payload, deliveryFor(payload)), + ).rejects.toThrow(/workflow.*(selection|configur|reader)|readBuildWorkflowPaths/i) + expect(enqueueWorkflowRunImport).not.toHaveBeenCalled() + expect(await prisma.gitHubWebhookDelivery.count()).toBe(0) + }) + + it("leaves configuration failures visible and unrecorded for later redelivery", async () => { + await createRepository(1001, "invalid-workflows") + let unavailable = true + const enqueueWorkflowRunImport = vi.fn(() => Promise.resolve()) + const dependencies = createGitHubWebhookRouteDependencies({ + installationId: "700", + provider: githubProvider, + syncRepositories: () => Promise.resolve(), + webhookSecret: "webhook-secret-for-tests", + enqueueWorkflowRunImport, + readBuildWorkflowPaths: () => + unavailable + ? Promise.reject(new Error(".github/replica.json builds.workflows is invalid.")) + : Promise.resolve([".github/workflows/publish.yml"]), + }) + const payload = { + action: "completed", + repository: { id: 1001 }, + workflow_run: { id: 55, path: ".github/workflows/publish.yml" }, + } + const delivery = deliveryFor(payload) + + await expect(dependencies.handleEvent("workflow_run", payload, delivery)).rejects.toThrow( + "builds.workflows", + ) + expect(enqueueWorkflowRunImport).not.toHaveBeenCalled() + expect(await prisma.gitHubWebhookDelivery.count()).toBe(0) + + unavailable = false + await expect(dependencies.handleEvent("workflow_run", payload, delivery)).resolves.toBe(true) + expect(enqueueWorkflowRunImport).toHaveBeenCalledOnce() + expect(await prisma.gitHubWebhookDelivery.count()).toBe(1) + }) + it("applies simultaneous copies of one delivery only once", async () => { const revisionId = await createProvider("github") const repository = await createRepository(1001, "accessible") diff --git a/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.ts b/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.ts index 33746af94..a9268d138 100644 --- a/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.ts +++ b/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.ts @@ -1,11 +1,11 @@ import { z } from "zod" import type { GitHubWorkflowRunJob } from "../../../../../domains/github-builds/get-github-workflow-run-job-definition/index.ts" -import { replicaWorkflowPath } from "../../../../../domains/github-builds/constants.ts" import { revokeGitHubRepositoryAccess } from "../../../../../domains/admission/revoke-github-repository-access/index.ts" import { recordGitHubBranchDeletion } from "../../../../../domains/prototypes/record-github-branch-deletion/index.ts" import { prisma } from "../../../../../integrations/prisma/index.ts" import { claimGitHubWebhookDelivery } from "../../../../../repositories/github-webhook-deliveries/claim-github-webhook-delivery/index.ts" +import { findConnectedGitHubRepository } from "../../../../../repositories/repository-access/find-connected-github-repository/index.ts" import type { GitHubWebhookRouteDependencies } from "../types.ts" const githubId = z.number().int().positive() @@ -81,6 +81,7 @@ export function createGitHubWebhookRouteDependencies(input: { syncRepositories: () => Promise /** Enqueues idempotently by key, so a redelivered event cannot import twice. */ enqueueWorkflowRunImport?: (job: GitHubWorkflowRunJob, key: string) => Promise + readBuildWorkflowPaths?: (githubRepositoryId: number) => Promise }): GitHubWebhookRouteDependencies { async function syncRepositories() { await input.syncRepositories() @@ -138,10 +139,14 @@ export function createGitHubWebhookRouteDependencies(input: { let job: GitHubWorkflowRunJob if (name === "workflow_run") { const event = workflowRunEvent.safeParse(payload) - if ( - !event.success || - event.data.workflow_run.path.split("@")[0] !== replicaWorkflowPath - ) { + if (!event.success) return false + const githubRepositoryId = event.data.repository.id + if (!(await findConnectedGitHubRepository(prisma, githubRepositoryId))) return false + if (!input.readBuildWorkflowPaths) { + throw new Error("GitHub Build workflow configuration reader is required.") + } + const paths = await input.readBuildWorkflowPaths(githubRepositoryId) + if (!paths.includes(event.data.workflow_run.path.split("@")[0] ?? "")) { return false } job = { githubRepositoryId: event.data.repository.id, runId: event.data.workflow_run.id } diff --git a/hub/server/utils/read-build-workflow-paths/index.test.ts b/hub/server/utils/read-build-workflow-paths/index.test.ts new file mode 100644 index 000000000..e25d81759 --- /dev/null +++ b/hub/server/utils/read-build-workflow-paths/index.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "vitest" + +import { readBuildWorkflowPaths } from "." + +describe("readBuildWorkflowPaths()", () => { + it("uses the legacy workflow when builds or workflows is omitted without validating other metadata", () => { + for (const configuration of [ + { version: 1 }, + { version: 1, builds: {} }, + { version: 1, hub: null, templates: "validated by the template consumer" }, + ]) { + expect(readBuildWorkflowPaths(configuration)).toEqual([".github/workflows/replica.yml"]) + } + }) + + it("selects only the configured direct workflow files, including both extensions and the path limit", () => { + const prefix = ".github/workflows/" + const longest = `${prefix}${"a".repeat(255 - prefix.length - 4)}.yml` + const workflows = [ + ".github/workflows/Publish_proto-1.2.yaml", + longest, + ...Array.from({ length: 8 }, (_, index) => `${prefix}build-${index}.yml`), + ] + + expect(readBuildWorkflowPaths({ version: 1, builds: { workflows } })).toEqual(workflows) + }) + + it.each([null, [], "configuration", {}, { version: 2 }, { version: 1, unexpected: true }])( + "rejects an unsupported configuration rather than selecting the legacy workflow: %j", + (value) => { + expect(() => readBuildWorkflowPaths(value)).toThrow(/version|configuration/i) + }, + ) + + it.each([ + null, + [], + "builds", + { workflow: ".github/workflows/build.yml" }, + { workflows: null }, + { workflows: ".github/workflows/build.yml" }, + { workflows: [] }, + { workflows: Array.from({ length: 11 }, (_, index) => `.github/workflows/${index}.yml`) }, + { workflows: [".github/workflows/build.yml", ".github/workflows/build.yml"] }, + ])("rejects invalid explicit build selection with an actionable error: %j", (builds) => { + expect(() => readBuildWorkflowPaths({ version: 1, builds })).toThrow(/builds|workflows/i) + }) + + it.each([ + 42, + "", + ".github/workflows/../build.yml", + ".github/workflows/nested/build.yml", + ".github/workflows/build.yml@refs/heads/proto/test", + ".github/workflows/build.yml?ref=main", + ".github/workflows/build.yml#main", + "https://github.com/o/r/.github/workflows/build.yml", + "refs/heads/main:.github/workflows/build.yml", + "/.github/workflows/build.yml", + ".github\\workflows\\build.yml", + ".github/workflows/%2e%2e.yml", + ".github/workflows/.hidden.yml", + ".github/workflows/-build.yml", + ".github/workflows/build file.yml", + ".github/workflows/build.YML", + ".github/workflows/build.json", + ".github/workflows/build.yml\n", + `.github/workflows/${"a".repeat(235)}.yml`, + ])("rejects a workflow path outside the direct-file contract: %j", (workflow) => { + expect(() => readBuildWorkflowPaths({ version: 1, builds: { workflows: [workflow] } })).toThrow( + /workflows/i, + ) + }) +}) diff --git a/hub/server/utils/read-build-workflow-paths/index.ts b/hub/server/utils/read-build-workflow-paths/index.ts new file mode 100644 index 000000000..405993f48 --- /dev/null +++ b/hub/server/utils/read-build-workflow-paths/index.ts @@ -0,0 +1,32 @@ +import { z } from "zod" + +const workflowPath = z + .string() + .max(255) + .regex(/^\.github\/workflows\/[A-Za-z0-9][A-Za-z0-9_.-]*\.ya?ml$/) +const configuration = z.strictObject({ + version: z.literal(1), + hub: z.unknown().optional(), + templates: z.unknown().optional(), + builds: z + .strictObject({ + workflows: z.array(workflowPath).min(1).max(10).optional(), + }) + .optional(), +}) + +/** Reads the repository's workflow selection without treating metadata as templates. */ +export function readBuildWorkflowPaths(value: unknown): string[] { + const parsed = configuration.safeParse(value) + if (!parsed.success) { + throw new Error( + ".github/replica.json must be a version 1 configuration; builds.workflows must contain 1 to 10 unique .github/workflows/.yml or .yaml paths (at most 255 characters each).", + { cause: parsed.error }, + ) + } + const paths = parsed.data.builds?.workflows ?? [".github/workflows/replica.yml"] + if (new Set(paths).size !== paths.length) { + throw new Error(".github/replica.json builds.workflows must not contain duplicate paths.") + } + return paths +} diff --git a/hub/server/workers/github-recovery.test.ts b/hub/server/workers/github-recovery.test.ts index e27265c43..95bbd300a 100644 --- a/hub/server/workers/github-recovery.test.ts +++ b/hub/server/workers/github-recovery.test.ts @@ -56,6 +56,7 @@ describe("GitHub recovery workers", () => { vi.stubEnv("GITHUB_WEBHOOK_SECRET", "test-secret-at-least-20-characters") vi.stubEnv("DATABASE_URL", "postgresql://localhost/test") vi.mocked(createGitHubActionsClient).mockImplementation(() => ({ + getBuildWorkflowPaths: vi.fn(() => Promise.resolve([".github/workflows/replica.yml"])), listBranchHeads: vi.fn(() => Promise.resolve([])), isAncestor: vi.fn(() => Promise.resolve(false)), getWorkflowRun: vi.fn(() => Promise.resolve(null)), @@ -81,4 +82,73 @@ describe("GitHub recovery workers", () => { expect.objectContaining({ githubRepositoryId: 2 }), ) }) + + it("uses configured workflow discovery and refreshes the selection for the next recovery poll", async () => { + clearGitHubSettings() + const { privateKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }) + vi.stubEnv("GITHUB_APP_ID", "123") + vi.stubEnv("GITHUB_APP_INSTALLATION_ID", "456") + vi.stubEnv( + "GITHUB_APP_PRIVATE_KEY_BASE64", + Buffer.from(privateKey.export({ type: "pkcs8", format: "pem" })).toString("base64"), + ) + vi.stubEnv("GITHUB_WEBHOOK_SECRET", "test-secret-at-least-20-characters") + vi.stubEnv("DATABASE_URL", "postgresql://localhost/test") + const actual = await vi.importActual<{ + createGitHubActionsClient: typeof createGitHubActionsClient + }>("../integrations/github/create-github-actions-client") + let filename = "publish-proto.yml" + const request = vi.fn((input) => { + const url = new URL(input instanceof Request ? input.url : input.toString()) + if (url.pathname.endsWith("/access_tokens")) { + return Promise.resolve(Response.json({ token: "test-token" })) + } + if (url.pathname === "/repositories/56") { + return Promise.resolve(Response.json({ id: 56, default_branch: "trunk" })) + } + if (url.pathname === "/repositories/56/git/ref/heads/trunk") { + return Promise.resolve(Response.json({ object: { type: "commit", sha: "c".repeat(40) } })) + } + if (url.pathname.endsWith("/replica.json")) { + const content = JSON.stringify({ + version: 1, + builds: { workflows: [`.github/workflows/${filename}`] }, + }) + return Promise.resolve( + Response.json({ + type: "file", + encoding: "base64", + size: Buffer.byteLength(content), + content: Buffer.from(content).toString("base64"), + }), + ) + } + expect(url.pathname).toBe(`/repositories/56/actions/workflows/${filename}/runs`) + return Promise.resolve(Response.json({ workflow_runs: [] })) + }) + vi.mocked(createGitHubActionsClient).mockImplementation((input) => + actual.createGitHubActionsClient(input, { fetch: request }), + ) + vi.mocked(discoverGitHubWorkflowRuns).mockImplementation(async (dependencies) => { + await dependencies.listWorkflowRuns(56, new Date("2026-01-01T00:00:00Z")) + return [] + }) + + await discoverGitHubWorkflowRunsJob() + filename = "preview.yaml" + await discoverGitHubWorkflowRunsJob() + + expect(createGitHubActionsClient).toHaveBeenCalledTimes(2) + const requests = request.mock.calls.map( + ([input]) => new URL(input instanceof Request ? input.url : input.toString()), + ) + expect( + requests + .filter((url) => url.pathname.includes("/actions/workflows/")) + .map((url) => url.pathname), + ).toEqual([ + "/repositories/56/actions/workflows/publish-proto.yml/runs", + "/repositories/56/actions/workflows/preview.yaml/runs", + ]) + }) }) From 9b030400819cecb3ea513df206bce6b5f5a741ff Mon Sep 17 00:00:00 2001 From: Will Eastler <32074613+weastler@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:34:36 -0800 Subject: [PATCH 2/4] fix(hub): verify workflow configuration tree entries Reject symbolic-link configuration sources before GitHub dereferences their contents, and distinguish missing files from provider failures at a pinned revision. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../scripts/validate-templates/index.test.mjs | 15 ++ hub/docs/github-actions-builds.md | 2 +- .../index.test.ts | 175 +++++++++++++++++- .../create-github-actions-client/index.ts | 50 ++++- hub/server/workers/github-recovery.test.ts | 16 ++ 5 files changed, 255 insertions(+), 3 deletions(-) diff --git a/actions/build/scripts/validate-templates/index.test.mjs b/actions/build/scripts/validate-templates/index.test.mjs index 7a6f98d21..bb22c9ffa 100644 --- a/actions/build/scripts/validate-templates/index.test.mjs +++ b/actions/build/scripts/validate-templates/index.test.mjs @@ -26,6 +26,21 @@ describe("validateTemplates()", () => { assert.equal(validateTemplates(metadata), null) }) + it("rejects malformed explicit build selection even when no templates are configured", () => { + for (const builds of [ + null, + [], + { workflow: ".github/workflows/build.yml" }, + { workflows: [] }, + { workflows: ".github/workflows/build.yml" }, + { workflows: [".github/workflows/nested/build.yml"] }, + { workflows: [".github/workflows/build.yml@refs/heads/main"] }, + { workflows: [".github/workflows/build.yml", ".github/workflows/build.yml"] }, + ]) { + assert.throws(() => validateTemplates({ version: 1, builds }), /builds|workflows/i) + } + }) + it("preserves local paths, query strings and fragments but removes Hub metadata", () => { assert.deepEqual( validateTemplates({ diff --git a/hub/docs/github-actions-builds.md b/hub/docs/github-actions-builds.md index 52c9a8254..ee9122bb1 100644 --- a/hub/docs/github-actions-builds.md +++ b/hub/docs/github-actions-builds.md @@ -25,7 +25,7 @@ Set `builds.workflows` in `.github/replica.json` on the connected repository's a Keep existing `hub` and `templates` fields alongside `builds`. The build action validates the new field but excludes it, like `hub`, from packaged template metadata. -Hub reads the default branch from GitHub, resolves its commit, and reads configuration at that exact revision. It never uses the prototype branch's configuration, so existing prototype branches need no configuration update. Webhook intake, import workers, and recovery use the same validated selection. Each event, import job, or repository poll reads a fresh selection; changing the file affects subsequent work without replacing or deleting successful Builds. +Hub reads the default branch from GitHub, resolves its commit, and checks its Git tree before reading configuration at that exact revision. Neither `.github` nor `replica.json` may be a symbolic link. Incomplete directory listings stop selection rather than count as missing configuration. Hub never uses the prototype branch's configuration, so existing prototype branches need no configuration update. Webhook intake, import workers, and recovery use the same validated selection. Each event, import job, or repository poll reads a fresh selection; changing the file affects subsequent work without replacing or deleting successful Builds. An omitted file, `builds`, or `builds.workflows` keeps the legacy default. An explicit list replaces that default; include `.github/workflows/replica.yml` explicitly to select it too. Use one to ten unique paths directly under `.github/workflows/`, ending in `.yml` or `.yaml`. Filenames must start with an ASCII letter or digit and contain only letters, digits, dots, underscores, or hyphens; each complete path is limited to 255 characters. Configuration must be a regular JSON file of at most 16 KiB. Invalid configuration stops imports and discovery with an error naming the configuration file; provider failures do not select a fallback workflow. diff --git a/hub/server/integrations/github/create-github-actions-client/index.test.ts b/hub/server/integrations/github/create-github-actions-client/index.test.ts index 556d33f55..fb3f2f285 100644 --- a/hub/server/integrations/github/create-github-actions-client/index.test.ts +++ b/hub/server/integrations/github/create-github-actions-client/index.test.ts @@ -20,6 +20,7 @@ function respond( if ( href.pathname === "/repositories/56" || href.pathname.startsWith("/repositories/56/git/ref/heads/") || + href.pathname.startsWith("/repositories/56/git/trees/") || href.pathname === "/repositories/56/contents/.github/replica.json" ) { const overridden = configuration?.(href, init ?? {}) @@ -30,6 +31,24 @@ function respond( if (href.pathname.includes("/git/ref/")) { return Promise.resolve(Response.json({ object: { type: "commit", sha: "c".repeat(40) } })) } + if (href.pathname === `/repositories/56/git/trees/${"c".repeat(40)}`) { + return Promise.resolve( + Response.json({ + truncated: false, + tree: [{ path: ".github", mode: "040000", type: "tree", sha: "d".repeat(40) }], + }), + ) + } + if (href.pathname === `/repositories/56/git/trees/${"d".repeat(40)}`) { + return Promise.resolve( + Response.json({ + truncated: false, + tree: configuration + ? [{ path: "replica.json", mode: "100644", type: "blob", sha: "e".repeat(40) }] + : [], + }), + ) + } return Promise.resolve(new Response("{}", { status: 404 })) } return Promise.resolve(handler(href, init ?? {})) @@ -102,6 +121,12 @@ describe("createGitHubActionsClient", () => { const configReads = requests.filter((url) => url.pathname.endsWith("/replica.json")) expect(configReads).toHaveLength(1) expect(configReads[0]?.searchParams.get("ref")).toBe("c".repeat(40)) + expect( + requests.filter((url) => url.pathname.includes("/git/trees/")).map((url) => url.pathname), + ).toEqual([ + `/repositories/56/git/trees/${"c".repeat(40)}`, + `/repositories/56/git/trees/${"d".repeat(40)}`, + ]) expect( requests .filter((url) => url.pathname.includes("/actions/workflows/")) @@ -209,7 +234,7 @@ describe("createGitHubActionsClient", () => { ).resolves.toEqual([expect.objectContaining({ id: 7 })]) }) - it.each([401, 403, 429, 503])( + it.each([401, 403, 404, 429, 503])( "does not treat a configuration HTTP %s failure as omitted configuration", async (status) => { const fetch = respond( @@ -224,6 +249,154 @@ describe("createGitHubActionsClient", () => { }, ) + it.each(["configuration file", "configuration directory"])( + "rejects a symlinked %s before the Contents API can follow it", + async (target) => { + const fetch = respond( + () => Response.json({}), + (url) => { + const treeSha = target === "configuration file" ? "d" : "c" + if (url.pathname === `/repositories/56/git/trees/${treeSha.repeat(40)}`) { + return Response.json({ + truncated: false, + tree: [ + { + path: target === "configuration file" ? "replica.json" : ".github", + mode: "120000", + type: "blob", + sha: "e".repeat(40), + }, + ], + }) + } + if (url.pathname.endsWith("/replica.json")) { + return configurationFile( + '{"version":1,"builds":{"workflows":[".github/workflows/redirected.yml"]}}', + ) + } + return undefined + }, + ) + + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).rejects.toThrow() + expect(fetch.mock.calls.some(([url]) => (url as URL).pathname.includes("/contents/"))).toBe( + false, + ) + }, + ) + + it("accepts a regular executable configuration at the end of a 2000-entry tree", async () => { + const fetch = respond( + () => Response.json({}), + (url) => { + if (url.pathname === `/repositories/56/git/trees/${"d".repeat(40)}`) { + return Response.json({ + truncated: false, + tree: [ + ...Array.from({ length: 1999 }, (_, index) => ({ + path: `other-${index}`, + mode: "100644", + type: "blob", + sha: "e".repeat(40), + })), + { path: "replica.json", mode: "100755", type: "blob", sha: "e".repeat(40) }, + ], + }) + } + return url.pathname.endsWith("/replica.json") + ? configurationFile( + '{"version":1,"builds":{"workflows":[".github/workflows/publish.yml"]}}', + ) + : undefined + }, + ) + + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).resolves.toEqual([".github/workflows/publish.yml"]) + }) + + it.each([".github", "replica.json"])( + "uses the legacy workflow only when the pinned tree has no %s entry", + async (missing) => { + const fetch = respond( + () => Response.json({}), + (url) => { + const treeSha = missing === ".github" ? "c" : "d" + if (url.pathname === `/repositories/56/git/trees/${treeSha.repeat(40)}`) { + return Response.json({ truncated: false, tree: [] }) + } + return undefined + }, + ) + + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).resolves.toEqual([".github/workflows/replica.yml"]) + expect(fetch.mock.calls.some(([url]) => (url as URL).pathname.includes("/contents/"))).toBe( + false, + ) + }, + ) + + it.each(["truncated", "malformed", "too many entries", "oversized"])( + "rejects %s tree evidence rather than treating configuration as absent", + async (failure) => { + const fetch = respond( + () => Response.json({}), + (url) => { + if (url.pathname === `/repositories/56/git/trees/${"d".repeat(40)}`) { + if (failure === "truncated") return Response.json({ truncated: true, tree: [] }) + if (failure === "malformed") return Response.json({ truncated: false, tree: null }) + if (failure === "oversized") { + return Response.json({ + truncated: false, + tree: [], + padding: "x".repeat(1024 * 1024), + }) + } + return Response.json({ + truncated: false, + tree: Array.from({ length: 2001 }, (_, index) => ({ + path: `other-${index}`, + mode: "100644", + type: "blob", + sha: "e".repeat(40), + })), + }) + } + return undefined + }, + ) + + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).rejects.toThrow() + expect(fetch.mock.calls.some(([url]) => (url as URL).pathname.includes("/contents/"))).toBe( + false, + ) + }, + ) + + it.each([ + { treeSha: "c", status: 404 }, + { treeSha: "d", status: 503 }, + ])("does not fall back after a tree HTTP $status failure", async ({ treeSha, status }) => { + const fetch = respond( + () => Response.json({}), + (url) => + url.pathname === `/repositories/56/git/trees/${treeSha.repeat(40)}` + ? new Response("{}", { status }) + : undefined, + ) + + await expect( + createGitHubActionsClient(input, { fetch }).getBuildWorkflowPaths(), + ).rejects.toThrow(GitHubProviderRequestError) + }) + it.each(["/repositories/56", "/repositories/56/git/ref/heads/trunk"])( "does not treat missing source evidence at %s as omitted configuration", async (path) => { diff --git a/hub/server/integrations/github/create-github-actions-client/index.ts b/hub/server/integrations/github/create-github-actions-client/index.ts index 1a5966080..339dbd870 100644 --- a/hub/server/integrations/github/create-github-actions-client/index.ts +++ b/hub/server/integrations/github/create-github-actions-client/index.ts @@ -77,6 +77,19 @@ const configurationFileSchema = z.looseObject({ size: z.number().int().nonnegative().max(maximumConfigurationBytes), content: z.string(), }) +const treeSchema = z.looseObject({ + truncated: z.literal(false), + tree: z + .array( + z.looseObject({ + path: z.string().max(255), + mode: z.string(), + type: z.string(), + sha: z.string().regex(/^[0-9a-f]{40}$/), + }), + ) + .max(2_000), +}) function mapRun(run: z.infer): GitHubWorkflowRun { const actor = run.triggering_actor ?? run.actor @@ -195,11 +208,46 @@ export function createGitHubActionsClient( authorization, ), ) + async function readTree(sha: string) { + const response = await send(`${repositoryPath}/git/trees/${sha}`, authorization) + if (!response.ok) throw new GitHubProviderRequestError() + const bytes = await readBounded(response, 1_048_576) + if (!bytes) { + throw new Error( + "Cannot read .github/replica.json: its configuration directory exceeds the 1 MiB discovery limit.", + ) + } + let value: unknown + try { + value = JSON.parse(bytes.toString("utf8")) + } catch (error) { + throw new GitHubProviderRequestError({ cause: error }) + } + const tree = treeSchema.safeParse(value) + if (!tree.success) { + throw new Error( + "Cannot read .github/replica.json: GitHub must return a complete configuration directory with at most 2,000 entries.", + { cause: tree.error }, + ) + } + return tree.data.tree + } + const directory = (await readTree(reference.object.sha)).find( + (entry) => entry.path === ".github", + ) + if (!directory) return readBuildWorkflowPaths({ version: 1 }) + if (directory.type !== "tree" || directory.mode !== "040000") { + throw new Error(".github must be a directory without symbolic links.") + } + const entry = (await readTree(directory.sha)).find((entry) => entry.path === "replica.json") + if (!entry) return readBuildWorkflowPaths({ version: 1 }) + if (entry.type !== "blob" || !["100644", "100755"].includes(entry.mode)) { + throw new Error(".github/replica.json must be a regular file without symbolic links.") + } const response = await send( `${repositoryPath}/contents/.github/replica.json?ref=${reference.object.sha}`, authorization, ) - if (response.status === 404) return readBuildWorkflowPaths({ version: 1 }) if (!response.ok) throw new GitHubProviderRequestError() const bytes = await readBounded(response, 32_768) if (!bytes) { diff --git a/hub/server/workers/github-recovery.test.ts b/hub/server/workers/github-recovery.test.ts index 95bbd300a..7ad032a73 100644 --- a/hub/server/workers/github-recovery.test.ts +++ b/hub/server/workers/github-recovery.test.ts @@ -109,6 +109,22 @@ describe("GitHub recovery workers", () => { if (url.pathname === "/repositories/56/git/ref/heads/trunk") { return Promise.resolve(Response.json({ object: { type: "commit", sha: "c".repeat(40) } })) } + if (url.pathname === `/repositories/56/git/trees/${"c".repeat(40)}`) { + return Promise.resolve( + Response.json({ + truncated: false, + tree: [{ path: ".github", mode: "040000", type: "tree", sha: "d".repeat(40) }], + }), + ) + } + if (url.pathname === `/repositories/56/git/trees/${"d".repeat(40)}`) { + return Promise.resolve( + Response.json({ + truncated: false, + tree: [{ path: "replica.json", mode: "100644", type: "blob", sha: "e".repeat(40) }], + }), + ) + } if (url.pathname.endsWith("/replica.json")) { const content = JSON.stringify({ version: 1, From b5542a268b4ba8edf7369309147249368305f176 Mon Sep 17 00:00:00 2001 From: Will Eastler <32074613+weastler@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:54:45 -0800 Subject: [PATCH 3/4] fix(hub): resolve build workflow review feedback Keep provider reads out of webhook acknowledgements, stop deselected unfinished Builds, and recover screenshots per contributing run without changing dist ownership. Document compatible action pins. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- hub/docs/database.md | 2 + hub/docs/github-actions-builds.md | 14 +- hub/schema/build.prisma | 1 + hub/schema/github-build-run-completion.prisma | 20 ++ .../migration.sql | 18 + .../index.db.test.ts | 7 +- .../discover-github-workflow-runs/index.ts | 7 +- .../index.db.test.ts | 332 +++++++++++++++++- .../import-github-workflow-run/index.ts | 18 +- .../is-github-workflow-branch-push/index.ts | 15 + .../index.ts | 13 +- hub/server/index.ts | 7 - .../list-github-build-runs/index.ts | 8 + .../index.db.test.ts | 152 ++++++++ .../index.ts | 41 ++- .../settle-deselected-github-build/index.ts | 51 +++ .../index.db.test.ts | 198 +++-------- .../index.ts | 12 +- .../index.db.test.ts | 196 +++++++++++ 19 files changed, 906 insertions(+), 206 deletions(-) create mode 100644 hub/schema/github-build-run-completion.prisma create mode 100644 hub/schema/migrations/20261218120000_github_build_run_completions/migration.sql create mode 100644 hub/server/domains/github-builds/is-github-workflow-branch-push/index.ts create mode 100644 hub/server/repositories/github-builds/record-github-build-run-completion/index.db.test.ts create mode 100644 hub/server/repositories/github-builds/settle-deselected-github-build/index.ts create mode 100644 hub/test/database/github-build-run-completion-migration/index.db.test.ts diff --git a/hub/docs/database.md b/hub/docs/database.md index 9ab05b805..08c020a45 100644 --- a/hub/docs/database.md +++ b/hub/docs/database.md @@ -86,6 +86,8 @@ Builds retain the author's provider, issuer, and provider user ID independently Builds created before commit metadata was stored retain `NULL` for unknown values during the upgrade. A later detail API must represent missing values rather than inventing them. This migration does not import Rails records or map their integer IDs. +GitHub Build run completions record the latest processed attempt for each contributing run on a successful Build. Recovery uses these receipts to retry another selected workflow's screenshot pass without transferring dist ownership. New receipts do not backfill or change existing Builds, artifact records, or owning-run completion times. + Review threads bind one non-removed repository, one reviewable prototype, and one successful Build. The root and replies keep stable Hub user UUIDs plus presentation snapshots. Soft deletion records when the author deleted a message but retains its source body; product reads and learning snapshots replace that body with `null`. Resolution events are immutable and uniquely ordered by their thread activity sequence, so resolve, reopen, and resolved-again transitions remain distinguishable for product reads and learning snapshots. Decision-model tables store repository settings, pinned input digests, runtime job and candidate references, promotion generations, Build evaluation references, and decimal usage or cost metadata. They do not store credentials, screenshots, tasksets, adapters, evaluation reports, or model weights. Database checks enforce training bounds, positive promotion generations, nonnegative usage and cost, and complete quantity/unit and amount/currency pairs. diff --git a/hub/docs/github-actions-builds.md b/hub/docs/github-actions-builds.md index ee9122bb1..0a925eb2e 100644 --- a/hub/docs/github-actions-builds.md +++ b/hub/docs/github-actions-builds.md @@ -23,9 +23,11 @@ Set `builds.workflows` in `.github/replica.json` on the connected repository's a } ``` -Keep existing `hub` and `templates` fields alongside `builds`. The build action validates the new field but excludes it, like `hub`, from packaged template metadata. +Before adding `builds`, upgrade every selected workflow's Replica publish, screenshots, and optional build actions to the same reviewed revision that supports the field. Earlier revisions, including `5ec2d4ad6d5343b61684ba2c7a1c8c82e2914565`, reject it during packaging. The example below pins `9b030400819cecb3ea513df206bce6b5f5a741ff`, which includes this support. Keep existing `hub` and `templates` fields alongside `builds`; supported actions validate the new field but exclude it, like `hub`, from packaged template metadata. -Hub reads the default branch from GitHub, resolves its commit, and checks its Git tree before reading configuration at that exact revision. Neither `.github` nor `replica.json` may be a symbolic link. Incomplete directory listings stop selection rather than count as missing configuration. Hub never uses the prototype branch's configuration, so existing prototype branches need no configuration update. Webhook intake, import workers, and recovery use the same validated selection. Each event, import job, or repository poll reads a fresh selection; changing the file affects subsequent work without replacing or deleting successful Builds. +Hub reads the default branch from GitHub, resolves its commit, and checks its Git tree before reading configuration at that exact revision. Neither `.github` nor `replica.json` may be a symbolic link. Incomplete directory listings stop selection rather than count as missing configuration. Hub never uses the prototype branch's configuration, so existing prototype branches need no configuration update. Webhooks only queue run IDs; import workers and recovery enforce the same validated selection. Each import job or repository poll reads a fresh selection; changing the file affects subsequent work without replacing or deleting successful Builds. + +Removing a workflow stops artifact imports from it. Its next import pass marks any queued or running Build still owned by that run as failed with a configuration explanation, so it leaves the awaiting-import sweep. It does not create Builds, transfer ownership, or change successful Builds. To retry a stopped Build, select the workflow again and rerun it as a new run or attempt. An omitted file, `builds`, or `builds.workflows` keeps the legacy default. An explicit list replaces that default; include `.github/workflows/replica.yml` explicitly to select it too. Use one to ten unique paths directly under `.github/workflows/`, ending in `.yml` or `.yaml`. Filenames must start with an ASCII letter or digit and contain only letters, digits, dots, underscores, or hyphens; each complete path is limited to 255 characters. Configuration must be a regular JSON file of at most 16 KiB. Invalid configuration stops imports and discovery with an error naming the configuration file; provider failures do not select a fallback workflow. @@ -63,7 +65,7 @@ jobs: node-version: 24 - run: npm ci - run: npm run build - - uses: github/replica/actions/build/publish@5ec2d4ad6d5343b61684ba2c7a1c8c82e2914565 + - uses: github/replica/actions/build/publish@9b030400819cecb3ea513df206bce6b5f5a741ff screenshots: needs: publish @@ -72,7 +74,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: github/replica/actions/build/screenshots@5ec2d4ad6d5343b61684ba2c7a1c8c82e2914565 + - uses: github/replica/actions/build/screenshots@9b030400819cecb3ea513df206bce6b5f5a741ff ``` The publish action validates and packages the output and uploads `replica-build-v1`. It installs no npm packages and no browser. Its completed-job event queues the Hub import while the dependent screenshot job downloads that exact artifact and uploads `replica-screenshots-v1`. The preview becomes available after webhook delivery, queue processing, and artifact import, not immediately on upload. If the build or artifact upload fails, GitHub skips screenshots and Hub records the failed Build. @@ -106,7 +108,7 @@ sequenceDiagram Q->>I: download replica-screenshots-v1 → attach if digest matches ``` -The webhook accepts `workflow_run` (`requested`, `in_progress`, `completed`) for selected workflow paths, and every completed `workflow_job`, because job events are the earliest signal that the dist artifact exists. The import worker checks the selected caller path and push event again using GitHub's run record. `workflow_run.completed` arrives only after every job, including screenshots, so it cannot provide early preview availability. +For connected repositories, the webhook queues `workflow_run` (`requested`, `in_progress`, `completed`) and every completed `workflow_job`, because job events are the earliest signal that the dist artifact exists. Intake makes no GitHub provider requests before acknowledging delivery. The import worker reads GitHub's run record, rejects ineligible events and branches before loading configuration, then checks the selected caller path. `workflow_run.completed` arrives only after every job, including screenshots, so it cannot provide early preview availability. The route enqueues before recording the delivery receipt. A queue failure therefore leaves the delivery unrecorded, so [recovery](#recovery) can redeliver it. Each delivery gets its own job ID, so an event that arrives while an earlier import of the same run is active still triggers another pass. @@ -123,7 +125,7 @@ One Build exists per prototype identity and commit. Each pass is idempotent: GitHub lists named artifacts across rerun attempts. Hub prefers packages created at or after the current attempt's `run_started_at`. If that attempt has no dist package, Hub reuses an unexpired dist from an earlier attempt of the same run only when the Build already has that digest or the current attempt's screenshots name it. This supports **Re-run failed jobs** when only screenshots failed and GitHub keeps the successful publish job's artifact. During a full rerun, the current attempt has no screenshots yet, so Hub waits for that attempt's own dist instead of fixing the Build to the earlier one. If the run completes without a matching dist, the Build fails without storing the earlier package. The first successful dist import fixes the Build's digest; subsequent attempts never replace it, and screenshots attach only when their digest matches that Build. -Hub records the completed workflow pass only after registration and screenshot processing finish. A transport or storage error leaves that pass pending for recovery even if the dist preview is already available. +Hub records each contributing run's completed attempt only after registration and screenshot processing finish. A transport or storage error leaves that run pending for recovery even if another run already imported the dist preview. A newer attempt still needs its own pass; delayed completion from an older attempt cannot undo a newer receipt. Owning-run completion and immutable dist ownership remain unchanged. Storage keys derive from the Build, file path, and content. Retries reuse identical objects, a key with different bytes stops the import, and keys are never deleted. Unreferenced keys left by an interrupted import are harmless. diff --git a/hub/schema/build.prisma b/hub/schema/build.prisma index 63fc951b3..5461786bc 100644 --- a/hub/schema/build.prisma +++ b/hub/schema/build.prisma @@ -11,6 +11,7 @@ model Build { reviewThreads ReviewThread[] shareLinks ShareLink[] decisionEvaluations DecisionBuildEvaluation[] + githubRunCompletions GitHubBuildRunCompletion[] // Attribute columns number Int diff --git a/hub/schema/github-build-run-completion.prisma b/hub/schema/github-build-run-completion.prisma new file mode 100644 index 000000000..ce5b0e954 --- /dev/null +++ b/hub/schema/github-build-run-completion.prisma @@ -0,0 +1,20 @@ +model GitHubBuildRunCompletion { + // ID columns + id String @id @default(uuid()) @db.Uuid + buildId String @map("build_id") @db.Uuid + + // Relations + build Build @relation(fields: [buildId], references: [id], onDelete: Restrict) + + // Attribute columns + githubWorkflowRunId BigInt @map("github_workflow_run_id") + runAttempt Int @map("run_attempt") + completedAt DateTime @map("completed_at") @db.Timestamptz(3) + + // Timestamp columns + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3) + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(3) + + @@unique([buildId, githubWorkflowRunId]) + @@map("github_build_run_completions") +} diff --git a/hub/schema/migrations/20261218120000_github_build_run_completions/migration.sql b/hub/schema/migrations/20261218120000_github_build_run_completions/migration.sql new file mode 100644 index 000000000..b3638f239 --- /dev/null +++ b/hub/schema/migrations/20261218120000_github_build_run_completions/migration.sql @@ -0,0 +1,18 @@ +-- CreateTable +CREATE TABLE "github_build_run_completions" ( + "id" UUID NOT NULL, + "build_id" UUID NOT NULL, + "github_workflow_run_id" BIGINT NOT NULL, + "run_attempt" INTEGER NOT NULL, + "completed_at" TIMESTAMPTZ(3) NOT NULL, + "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "github_build_run_completions_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "github_build_run_completions_build_id_github_workflow_run_i_key" ON "github_build_run_completions"("build_id", "github_workflow_run_id"); + +-- AddForeignKey +ALTER TABLE "github_build_run_completions" ADD CONSTRAINT "github_build_run_completions_build_id_fkey" FOREIGN KEY ("build_id") REFERENCES "builds"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/hub/server/domains/github-builds/discover-github-workflow-runs/index.db.test.ts b/hub/server/domains/github-builds/discover-github-workflow-runs/index.db.test.ts index 50a633143..b173fd3e9 100644 --- a/hub/server/domains/github-builds/discover-github-workflow-runs/index.db.test.ts +++ b/hub/server/domains/github-builds/discover-github-workflow-runs/index.db.test.ts @@ -228,7 +228,7 @@ describe("discoverGitHubWorkflowRuns", () => { ) }) - it("does not repeatedly enqueue runs that reuse a settled commit or lost Build ownership", async () => { + it("queues a successful commit's other caller but skips an older run that lost unfinished Build ownership", async () => { const repository = await createRepository() const settledOwner = workflowRun(repository.githubRepositoryId, { id: 100 }) const reused = { ...settledOwner, id: 102 } @@ -248,7 +248,10 @@ describe("discoverGitHubWorkflowRuns", () => { { createdAfter }, ) - expect(jobs).toEqual([{ githubRepositoryId: repository.githubRepositoryId, runId: owner.id }]) + expect(jobs).toEqual([ + { githubRepositoryId: repository.githubRepositoryId, runId: reused.id }, + { githubRepositoryId: repository.githubRepositoryId, runId: owner.id }, + ]) }) it("does not mistake another repository, branch, or commit for a settled Build", async () => { diff --git a/hub/server/domains/github-builds/discover-github-workflow-runs/index.ts b/hub/server/domains/github-builds/discover-github-workflow-runs/index.ts index 2ce650e4b..e7a0aea7a 100644 --- a/hub/server/domains/github-builds/discover-github-workflow-runs/index.ts +++ b/hub/server/domains/github-builds/discover-github-workflow-runs/index.ts @@ -30,7 +30,12 @@ function settled(run: GitHubWorkflowRun, saved: SavedRun[]) { if (build.branchName !== run.headBranch || build.commitSha !== run.headSha) return false if (build.state === "superseded" || build.branchDeletedAt !== null) return true if (build.runId !== run.id) { - return build.state === "successful" || (build.runId !== null && build.runId > run.id) + if (build.state === "successful") { + return build.runCompletions.some( + (completed) => completed.runId === run.id && completed.runAttempt >= run.runAttempt, + ) + } + return build.runId !== null && build.runId > run.id } return ( build.runAttempt >= run.runAttempt && diff --git a/hub/server/domains/github-builds/import-github-workflow-run/index.db.test.ts b/hub/server/domains/github-builds/import-github-workflow-run/index.db.test.ts index cbb447210..6aa4a692e 100644 --- a/hub/server/domains/github-builds/import-github-workflow-run/index.db.test.ts +++ b/hub/server/domains/github-builds/import-github-workflow-run/index.db.test.ts @@ -8,6 +8,7 @@ import type { GitHubWorkflowRun, } from "../../../integrations/github/create-github-actions-client/types.ts" import { findBuildArtifact } from "../../../repositories/build-artifacts/find-build-artifact/index.ts" +import { listGitHubBuildsAwaitingImport } from "../../../repositories/github-builds/list-github-builds-awaiting-import" import { createGitHubWebhookRouteDependencies } from "../../../routes/api/github/webhooks/create-github-webhook-route-dependencies" import { createZipArchive } from "../../../test/create-zip-archive/index.ts" import type { ArtifactStorage } from "../../artifacts/create-artifact-storage/types.ts" @@ -85,7 +86,7 @@ async function createFixture() { const hooks: { beforeDownload?: (id: number) => Promise } = {} const client = { getBuildWorkflowPaths: () => Promise.resolve([...workflowPaths]), - getWorkflowRun: () => Promise.resolve({ ...run }), + getWorkflowRun: (): Promise => Promise.resolve({ ...run }), listRunArtifacts: (_runId: number, name: string) => Promise.resolve( packages.has(name) @@ -157,7 +158,6 @@ describe("importGitHubWorkflowRun", () => { provider: { configurationDigest: "test-provider", revision: 1 }, webhookSecret: "webhook-secret-for-tests", syncRepositories: () => Promise.resolve(), - readBuildWorkflowPaths: () => fixture.client.getBuildWorkflowPaths(), enqueueWorkflowRunImport: (job) => { enqueued.push(job) return Promise.resolve() @@ -241,7 +241,7 @@ describe("importGitHubWorkflowRun", () => { expect(getRun).not.toHaveBeenCalled() }) - it("propagates configuration failures before reading or preparing a workflow run", async () => { + it("propagates configuration failures for an eligible run without preparing a Build", async () => { const fixture = await createFixture() vi.spyOn(fixture.client, "getBuildWorkflowPaths").mockRejectedValue( new Error(".github/replica.json builds.workflows is invalid."), @@ -249,24 +249,41 @@ describe("importGitHubWorkflowRun", () => { const getRun = vi.spyOn(fixture.client, "getWorkflowRun") await expect(fixture.importRun()).rejects.toThrow("builds.workflows") - expect(getRun).not.toHaveBeenCalled() + expect(getRun).toHaveBeenCalledWith(fixture.run.id) expect(await prisma.build.count()).toBe(0) }) it.each([ - { path: ".github/workflows/replica.yml" }, - { path: ".github/workflows/publish-proto.yml-extra" }, { event: "pull_request" }, + { event: "workflow_dispatch" }, + { event: "workflow_run" }, { headBranch: null }, { headBranch: "replica/install-workflow" }, { repositoryId: 0 }, { headSha: "not-an-immutable-commit" }, - ])("keeps existing run safety checks for custom workflow selection: %j", async (changes) => { + { htmlUrl: "http://github.com/octo/proto/actions/runs/1" }, + ])("ignores an ineligible run before reading invalid configuration: %j", async (changes) => { const fixture = await createFixture() fixture.workflowPaths.splice(0, 1, ".github/workflows/publish-proto.yml") Object.assign(fixture.run, { path: ".github/workflows/publish-proto.yml" }, changes) + const selection = vi + .spyOn(fixture.client, "getBuildWorkflowPaths") + .mockRejectedValue(new Error(".github/replica.json builds.workflows is invalid.")) + + await expect(fixture.importRun()).resolves.toEqual({ dist: "ignored", screenshots: "absent" }) + expect(selection).not.toHaveBeenCalled() + expect(await prisma.build.count()).toBe(0) + }) + + it("ignores a missing run before reading invalid configuration", async () => { + const fixture = await createFixture() + vi.spyOn(fixture.client, "getWorkflowRun").mockResolvedValue(null) + const selection = vi + .spyOn(fixture.client, "getBuildWorkflowPaths") + .mockRejectedValue(new Error(".github/replica.json builds.workflows is invalid.")) await expect(fixture.importRun()).resolves.toEqual({ dist: "ignored", screenshots: "absent" }) + expect(selection).not.toHaveBeenCalled() expect(await prisma.build.count()).toBe(0) }) @@ -464,6 +481,94 @@ describe("importGitHubWorkflowRun", () => { await expect(fixture.findAnalyticsEvents()).resolves.toEqual(events) }) + it("recovers another selected caller's screenshots until that run attempt finishes processing", async () => { + const fixture = await createFixture() + fixture.workflowPaths.push(".github/workflows/preview.yaml") + fixture.run.id = 100 + fixture.run.status = "completed" + fixture.run.conclusion = "success" + fixture.packages.set(distArtifactName, distPackage()) + await fixture.importRun() + const owner = await fixture.findBuild() + expect(owner.workflowCompletedAt).toEqual(fixture.run.updatedAt) + + fixture.run.id = 101 + fixture.run.path = ".github/workflows/preview.yaml" + fixture.run.createdAt = new Date("2026-01-01T00:02:00Z") + fixture.run.updatedAt = new Date("2026-01-01T00:03:00Z") + fixture.run.status = "in_progress" + fixture.run.conclusion = null + fixture.packages.set(distArtifactName, distPackage("

Another caller's dist

")) + const job = { githubRepositoryId: fixture.run.repositoryId, runId: fixture.run.id } + function poll() { + return discoverGitHubWorkflowRuns( + { + database: prisma, + listWorkflowRuns: () => Promise.resolve([fixture.run]), + listBranchHeads: () => Promise.resolve([]), + isAncestor: () => Promise.resolve(false), + logger, + }, + { createdAfter: new Date("2025-12-31T00:00:00Z") }, + ) + } + + await expect(poll()).resolves.toEqual([job]) + await expect(fixture.importRun()).resolves.toEqual({ + dist: "unchanged", + screenshots: "absent", + }) + await expect(poll()).resolves.toEqual([job]) + + fixture.run.status = "completed" + fixture.run.conclusion = "success" + fixture.run.updatedAt = new Date("2026-01-01T00:05:00Z") + fixture.packages.set(screenshotArtifactName, screenshotPackage(owner.distDigest!)) + fixture.hooks.beforeDownload = () => Promise.reject(new Error("Screenshots unavailable")) + await expect(fixture.importRun()).rejects.toThrow("Screenshots unavailable") + await expect(poll()).resolves.toEqual([job]) + + fixture.hooks.beforeDownload = undefined + await expect(fixture.importRun()).resolves.toEqual({ + dist: "unchanged", + screenshots: "attached", + }) + await expect(poll()).resolves.toEqual([]) + const completed = await fixture.findBuild() + expect(completed.artifacts.map((artifact) => artifact.path)).toContain( + "screenshots/desktop.png", + ) + + fixture.run.runAttempt = 2 + fixture.run.status = "in_progress" + fixture.run.conclusion = null + fixture.packages.delete(screenshotArtifactName) + await expect(poll()).resolves.toEqual([job]) + await expect(fixture.importRun()).resolves.toEqual({ + dist: "unchanged", + screenshots: "absent", + }) + await expect(poll()).resolves.toEqual([job]) + + fixture.run.status = "completed" + fixture.run.conclusion = "failure" + fixture.run.updatedAt = new Date("2026-01-01T00:10:00Z") + fixture.packages.set(screenshotArtifactName, screenshotPackage("sha256:" + "0".repeat(64))) + await expect(fixture.importRun()).resolves.toEqual({ + dist: "unchanged", + screenshots: "digest-mismatch", + }) + await expect(poll()).resolves.toEqual([]) + expect(await fixture.findBuild()).toMatchObject({ + state: "successful", + githubWorkflowRunId: owner.githubWorkflowRunId, + githubWorkflowRunAttempt: owner.githubWorkflowRunAttempt, + workflowCompletedAt: owner.workflowCompletedAt, + distDigest: owner.distDigest, + artifacts: completed.artifacts, + }) + }) + it("retries a newer screenshot attempt without reopening or rewriting a successful dist", async () => { const fixture = await createFixture() fixture.run.status = "completed" @@ -884,18 +989,209 @@ describe("importGitHubWorkflowRun", () => { }) }) - it("ignores runs that are not Replica push workflows", async () => { - const fixture = await createFixture() - fixture.run.path = ".github/workflows/ci.yml" - await expect(fixture.importRun()).resolves.toEqual({ dist: "ignored", screenshots: "absent" }) - await expect( - prisma.prototype.count({ - where: { - branchName: "prototype/welcome", - repository: { githubRepositoryId: fixture.run.repositoryId }, + it.each([".github/workflows/replica.yml", ".github/workflows/publish-proto.yml-extra"])( + "queues but does not prepare an unrelated workflow path %s", + async (path) => { + const fixture = await createFixture() + fixture.workflowPaths.splice(0, 1, ".github/workflows/publish-proto.yml") + fixture.run.path = path + const listArtifacts = vi.spyOn(fixture.client, "listRunArtifacts") + const download = vi.spyOn(fixture.client, "downloadArtifact") + const enqueued: GitHubWorkflowRunJob[] = [] + const webhooks = createGitHubWebhookRouteDependencies({ + installationId: "700", + provider: { configurationDigest: "test-provider", revision: 1 }, + webhookSecret: "webhook-secret-for-tests", + syncRepositories: () => Promise.resolve(), + enqueueWorkflowRunImport: (job) => { + enqueued.push(job) + return Promise.resolve() }, - }), - ).resolves.toBe(0) + }) + const payload = { + action: "completed", + repository: { id: fixture.run.repositoryId }, + workflow_run: { id: fixture.run.id, path: fixture.run.path }, + } + + await expect( + webhooks.handleEvent("workflow_run", payload, { + id: randomUUID(), + payloadDigest: createHash("sha256").update(JSON.stringify(payload)).digest("hex"), + }), + ).resolves.toBe(true) + expect(enqueued).toEqual([ + { githubRepositoryId: fixture.run.repositoryId, runId: fixture.run.id }, + ]) + await expect(fixture.importRun(enqueued[0])).resolves.toEqual({ + dist: "ignored", + screenshots: "absent", + }) + await expect( + prisma.prototype.count({ + where: { + branchName: "prototype/welcome", + repository: { githubRepositoryId: fixture.run.repositoryId }, + }, + }), + ).resolves.toBe(0) + expect(listArtifacts).not.toHaveBeenCalled() + expect(download).not.toHaveBeenCalled() + }, + ) + + it.each([ + { initialStatus: "queued", observedStatus: "in_progress", observedAttempt: 2 }, + { initialStatus: "in_progress", observedStatus: "completed", observedAttempt: 1 }, + ])( + "settles a deselected $initialStatus Build without importing artifacts and recovers only on a newer attempt", + async ({ initialStatus, observedStatus, observedAttempt }) => { + const fixture = await createFixture() + fixture.run.status = initialStatus + await fixture.importRun() + const before = await fixture.findBuild() + expect(before.state).toBe(initialStatus === "queued" ? "queued" : "running") + const createdAfter = new Date(0) + const job = { githubRepositoryId: fixture.run.repositoryId, runId: fixture.run.id } + await expect(listGitHubBuildsAwaitingImport(prisma, { createdAfter })).resolves.toEqual([job]) + const observations = await prisma.branchHeadObservation.findMany() + fixture.workflowPaths.splice(0) + fixture.run.status = observedStatus + fixture.run.runAttempt = observedAttempt + fixture.run.updatedAt = new Date("2026-01-01T00:05:00Z") + fixture.packages.set(distArtifactName, distPackage()) + const listArtifacts = vi.spyOn(fixture.client, "listRunArtifacts") + const download = vi.spyOn(fixture.client, "downloadArtifact") + const write = vi.spyOn(fixture.storage, "write") + + await expect(fixture.importRun()).resolves.toEqual({ + dist: "ignored", + screenshots: "absent", + }) + + const stopped = await fixture.findBuild() + expect(stopped).toMatchObject({ + id: before.id, + state: "failed", + workflowCompletedAt: observedStatus === "completed" ? fixture.run.updatedAt : null, + githubWorkflowRunId: before.githubWorkflowRunId, + commitSha: before.commitSha, + artifacts: [], + prototype: before.prototype, + }) + if (before.startedAt) expect(stopped.startedAt).toEqual(before.startedAt) + else expect(stopped.startedAt).toBeInstanceOf(Date) + expect(stopped.finishedAt).toBeInstanceOf(Date) + expect(stopped.failureSummary).toMatch(/default branch/i) + expect(stopped.failureSummary).toContain("builds.workflows") + expect(await prisma.branchHeadObservation.findMany()).toEqual(observations) + expect(listArtifacts).not.toHaveBeenCalled() + expect(download).not.toHaveBeenCalled() + expect(write).not.toHaveBeenCalled() + await expect(listGitHubBuildsAwaitingImport(prisma, { createdAfter })).resolves.toEqual([]) + + fixture.workflowPaths.push(".github/workflows/replica.yml") + await fixture.importRun() + expect(await fixture.findBuild()).toMatchObject({ + state: "failed", + failureSummary: stopped.failureSummary, + artifacts: [], + }) + expect(write).not.toHaveBeenCalled() + + fixture.run.runAttempt += 1 + fixture.run.status = "in_progress" + await expect(fixture.importRun()).resolves.toMatchObject({ dist: "imported" }) + expect(await fixture.findBuild()).toMatchObject({ + id: before.id, + state: "successful", + failureSummary: null, + githubWorkflowRunAttempt: fixture.run.runAttempt, + }) + }, + ) + + it.each([ + "older owner", + "newer owner", + "newer attempt", + "different commit", + "different branch", + "superseded Build", + "retired branch", + "disconnected repository", + ])("does not settle a deselected run's %s", async (boundary) => { + const fixture = await createFixture() + await fixture.importRun() + const build = await fixture.findBuild() + if (boundary === "older owner") { + fixture.run.id += 1 + } else if (boundary === "newer owner") { + await prisma.build.update({ + where: { id: build.id }, + data: { githubWorkflowRunId: BigInt(fixture.run.id + 1) }, + }) + } else if (boundary === "newer attempt") { + await prisma.build.update({ + where: { id: build.id }, + data: { githubWorkflowRunAttempt: fixture.run.runAttempt + 1 }, + }) + } else if (boundary === "different commit") { + fixture.run.headSha = "b".repeat(40) + } else if (boundary === "different branch") { + fixture.run.headBranch = "proto/another" + } else if (boundary === "superseded Build") { + await prisma.build.update({ + where: { id: build.id }, + data: { state: "superseded", startedAt: null, finishedAt: new Date() }, + }) + } else if (boundary === "retired branch") { + await prisma.prototype.update({ + where: { id: build.prototypeId }, + data: { branchDeletedAt: new Date() }, + }) + } else { + await prisma.repository.update({ + where: { githubRepositoryId: fixture.run.repositoryId }, + data: { removedAt: new Date() }, + }) + } + const before = await fixture.findBuild() + fixture.workflowPaths.splice(0) + const listArtifacts = vi.spyOn(fixture.client, "listRunArtifacts") + + await expect(fixture.importRun()).resolves.toEqual({ + dist: "ignored", + screenshots: "absent", + }) + + expect(await fixture.findBuild()).toEqual(before) + expect(await prisma.build.count()).toBe(1) + expect(listArtifacts).not.toHaveBeenCalled() + }) + + it("preserves a successful Build and its owner when its workflow is deselected", async () => { + const fixture = await createFixture() + fixture.run.status = "completed" + fixture.run.conclusion = "success" + fixture.packages.set(distArtifactName, distPackage()) + fixture.packages.set( + screenshotArtifactName, + screenshotPackage(extractDistPackage(distPackage()).distDigest), + ) + await fixture.importRun() + const before = await fixture.findBuild() + fixture.workflowPaths.splice(0) + fixture.run.runAttempt += 1 + const listArtifacts = vi.spyOn(fixture.client, "listRunArtifacts") + + await expect(fixture.importRun()).resolves.toEqual({ + dist: "ignored", + screenshots: "absent", + }) + + expect(await fixture.findBuild()).toEqual(before) + expect(listArtifacts).not.toHaveBeenCalled() }) it("does not move the current Build back for a late older run", async () => { diff --git a/hub/server/domains/github-builds/import-github-workflow-run/index.ts b/hub/server/domains/github-builds/import-github-workflow-run/index.ts index dfd01ae71..08dde8180 100644 --- a/hub/server/domains/github-builds/import-github-workflow-run/index.ts +++ b/hub/server/domains/github-builds/import-github-workflow-run/index.ts @@ -1,6 +1,7 @@ import { completeGitHubBuildFailure } from "../../../repositories/github-builds/complete-github-build-failure/index.ts" import { markGitHubBuildRunning } from "../../../repositories/github-builds/mark-github-build-running/index.ts" import { recordGitHubBuildRunCompletion } from "../../../repositories/github-builds/record-github-build-run-completion/index.ts" +import { settleDeselectedGitHubBuild } from "../../../repositories/github-builds/settle-deselected-github-build/index.ts" import { findConnectedGitHubRepository } from "../../../repositories/repository-access/find-connected-github-repository/index.ts" import { BuildPackageError } from "../../build-artifacts/errors.ts" import { extractScreenshotPackage } from "../../build-artifacts/extract-screenshot-package/index.ts" @@ -17,6 +18,7 @@ import { import type { GitHubWorkflowRunJob } from "../get-github-workflow-run-job-definition/index.ts" import { importGitHubDistPackage } from "../import-github-dist-package/index.ts" import { importGitHubScreenshotPackage } from "../import-github-screenshot-package/index.ts" +import { isGitHubWorkflowBranchPush } from "../is-github-workflow-branch-push/index.ts" import { prepareGitHubWorkflowRunBuild } from "../prepare-github-workflow-run-build/index.ts" import type { ImportGitHubWorkflowRunDependencies, ImportGitHubWorkflowRunResult } from "./types.ts" @@ -45,10 +47,22 @@ export async function importGitHubWorkflowRun( if (!(await findConnectedGitHubRepository(database, job.githubRepositoryId))) { return { dist: "ignored", screenshots: "absent" } } - const workflowPaths = await client.getBuildWorkflowPaths() const found = await client.getWorkflowRun(job.runId) - if (!found) return { dist: "ignored", screenshots: "absent" } + if (!found || !isGitHubWorkflowBranchPush(job.githubRepositoryId, found)) { + return { dist: "ignored", screenshots: "absent" } + } const run = found + const workflowPaths = await client.getBuildWorkflowPaths() + if (!workflowPaths.includes(run.path.split("@")[0] ?? "")) { + const settled = await settleDeselectedGitHubBuild(database, { + githubRepositoryId: job.githubRepositoryId, + run, + }) + if (settled) { + logger.info({ runId: run.id }, "Stopped unfinished Build from a deselected workflow") + } + return { dist: "ignored", screenshots: "absent" } + } const prepared = await prepareGitHubWorkflowRunBuild( database, job.githubRepositoryId, diff --git a/hub/server/domains/github-builds/is-github-workflow-branch-push/index.ts b/hub/server/domains/github-builds/is-github-workflow-branch-push/index.ts new file mode 100644 index 000000000..7b13f8339 --- /dev/null +++ b/hub/server/domains/github-builds/is-github-workflow-branch-push/index.ts @@ -0,0 +1,15 @@ +import type { GitHubWorkflowRun } from "../../../integrations/github/create-github-actions-client/types.ts" + +export function isGitHubWorkflowBranchPush( + githubRepositoryId: number, + run: GitHubWorkflowRun, +): run is GitHubWorkflowRun & { headBranch: string } { + return ( + run.repositoryId === githubRepositoryId && + run.event === "push" && + !!run.headBranch && + run.headBranch !== "replica/install-workflow" && + /^[0-9a-f]{40}$/.test(run.headSha) && + run.htmlUrl.startsWith("https://") + ) +} diff --git a/hub/server/domains/github-builds/prepare-github-workflow-run-build/index.ts b/hub/server/domains/github-builds/prepare-github-workflow-run-build/index.ts index 86d9a113e..3d8f932e0 100644 --- a/hub/server/domains/github-builds/prepare-github-workflow-run-build/index.ts +++ b/hub/server/domains/github-builds/prepare-github-workflow-run-build/index.ts @@ -1,6 +1,7 @@ import type { PrismaClient } from "../../../generated/prisma/client.ts" import type { GitHubWorkflowRun } from "../../../integrations/github/create-github-actions-client/types.ts" import { prepareGitHubBuild } from "../../../repositories/github-builds/prepare-github-build/index.ts" +import { isGitHubWorkflowBranchPush } from "../is-github-workflow-branch-push/index.ts" /** * Finds or creates the Build for a Replica workflow branch push run, using GitHub's @@ -12,15 +13,9 @@ export async function prepareGitHubWorkflowRunBuild( run: GitHubWorkflowRun, workflowPaths: string[], ) { - const branchName = run.headBranch if ( - run.repositoryId !== githubRepositoryId || - !workflowPaths.includes(run.path.split("@")[0] ?? "") || - run.event !== "push" || - !branchName || - branchName === "replica/install-workflow" || - !/^[0-9a-f]{40}$/.test(run.headSha) || - !run.htmlUrl.startsWith("https://") + !isGitHubWorkflowBranchPush(githubRepositoryId, run) || + !workflowPaths.includes(run.path.split("@")[0] ?? "") ) { return { status: "ignored" as const } } @@ -32,7 +27,7 @@ export async function prepareGitHubWorkflowRunBuild( } const prepared = await prepareGitHubBuild(database, { githubRepositoryId, - branchName, + branchName: run.headBranch, commitSha: run.headSha, run: buildRun, author: null, diff --git a/hub/server/index.ts b/hub/server/index.ts index f2e32b42f..ccae13415 100644 --- a/hub/server/index.ts +++ b/hub/server/index.ts @@ -218,8 +218,6 @@ try { const context = createGitHubAuthenticationContext(authenticationEnvironment) const { createGitHubWebhookRouteDependencies } = await import("./routes/api/github/webhooks/create-github-webhook-route-dependencies/index.ts") - const { createGitHubActionsClient } = - await import("./integrations/github/create-github-actions-client/index.ts") githubWebhooks = createGitHubWebhookRouteDependencies({ installationId: appCredentials.installationId, provider: { @@ -228,11 +226,6 @@ try { }, syncRepositories: () => setup.syncRepositories(), webhookSecret: appCredentials.webhookSecret, - readBuildWorkflowPaths: (githubRepositoryId) => - createGitHubActionsClient({ - ...appCredentials, - githubRepositoryId, - }).getBuildWorkflowPaths(), enqueueWorkflowRunImport: async (job, deliveryId) => { const { enqueueGitHubWorkflowRunImport } = await import("./domains/github-builds/enqueue-github-workflow-run-import/index.ts") diff --git a/hub/server/repositories/github-builds/list-github-build-runs/index.ts b/hub/server/repositories/github-builds/list-github-build-runs/index.ts index f2fc95b58..2de4f98c9 100644 --- a/hub/server/repositories/github-builds/list-github-build-runs/index.ts +++ b/hub/server/repositories/github-builds/list-github-build-runs/index.ts @@ -40,6 +40,10 @@ export async function listGitHubBuildRuns( githubWorkflowRunAttempt: true, state: true, workflowCompletedAt: true, + githubRunCompletions: { + where: { githubWorkflowRunId: { in: runs.map((run) => BigInt(run.id)) } }, + select: { githubWorkflowRunId: true, runAttempt: true }, + }, }, }) return builds @@ -55,5 +59,9 @@ export async function listGitHubBuildRuns( runAttempt: build.githubWorkflowRunAttempt ?? 0, state: build.state, workflowCompletedAt: build.workflowCompletedAt, + runCompletions: build.githubRunCompletions.map((run) => ({ + runId: Number(run.githubWorkflowRunId), + runAttempt: run.runAttempt, + })), })) } diff --git a/hub/server/repositories/github-builds/record-github-build-run-completion/index.db.test.ts b/hub/server/repositories/github-builds/record-github-build-run-completion/index.db.test.ts new file mode 100644 index 000000000..6a5f17d5f --- /dev/null +++ b/hub/server/repositories/github-builds/record-github-build-run-completion/index.db.test.ts @@ -0,0 +1,152 @@ +import { randomUUID } from "node:crypto" + +import { describe, expect, it } from "vitest" + +import type { BuildState } from "../../../generated/prisma/client" +import { prisma } from "../../../integrations/prisma" +import { recordGitHubBuildRunCompletion } from "." + +async function createBuild(state: BuildState = "successful") { + const repository = await prisma.repository.create({ + data: { nameWithOwner: `octo/${randomUUID()}`, defaultBranch: "main" }, + }) + const prototype = await prisma.prototype.create({ + data: { repositoryId: repository.id, branchName: "proto/welcome" }, + }) + return prisma.build.create({ + data: { + prototypeId: prototype.id, + number: 1, + state, + commitSha: "a".repeat(40), + githubWorkflowRunId: 100n, + githubWorkflowRunAttempt: 2, + distDigest: state === "successful" ? "sha256:" + "b".repeat(64) : null, + artifacts: + state === "successful" + ? { + create: { + kind: "dist", + path: "dist/index.html", + storageKey: "retained-dist", + byteLength: 10, + }, + } + : undefined, + }, + include: { artifacts: true }, + }) +} + +describe("recordGitHubBuildRunCompletion()", () => { + it("records another run's pass without marking the owning run complete", async () => { + const build = await createBuild() + const completedAt = new Date("2026-11-18T12:00:00Z") + + await expect( + recordGitHubBuildRunCompletion(prisma, { + buildId: build.id, + run: { runId: 101n, runAttempt: 1 }, + completedAt, + }), + ).resolves.toBe(true) + + expect( + await prisma.build.findUniqueOrThrow({ + where: { id: build.id }, + include: { artifacts: true }, + }), + ).toEqual(build) + expect( + await prisma.gitHubBuildRunCompletion.findMany({ where: { buildId: build.id } }), + ).toMatchObject([{ buildId: build.id, githubWorkflowRunId: 101n, runAttempt: 1, completedAt }]) + + await recordGitHubBuildRunCompletion(prisma, { + buildId: build.id, + run: { runId: 100n, runAttempt: 1 }, + completedAt, + }) + expect( + (await prisma.build.findUniqueOrThrow({ where: { id: build.id } })).workflowCompletedAt, + ).toBeNull() + + await recordGitHubBuildRunCompletion(prisma, { + buildId: build.id, + run: { runId: 100n, runAttempt: 2 }, + completedAt, + }) + const completed = await prisma.build.findUniqueOrThrow({ + where: { id: build.id }, + include: { artifacts: true }, + }) + expect(completed).toEqual({ + ...build, + workflowCompletedAt: completedAt, + updatedAt: completed.updatedAt, + }) + }) + + it("retains one completion at the newest attempt across concurrent and stale passes", async () => { + const build = await createBuild() + const completedAt = new Date("2026-11-18T12:03:00Z") + await Promise.all( + [2, 3, 1, 3].map((runAttempt) => + recordGitHubBuildRunCompletion(prisma, { + buildId: build.id, + run: { runId: 101n, runAttempt }, + completedAt: runAttempt === 3 ? completedAt : new Date("2026-11-18T12:01:00Z"), + }), + ), + ) + const [newest] = await prisma.gitHubBuildRunCompletion.findMany({ + where: { buildId: build.id }, + }) + expect(newest).toMatchObject({ + buildId: build.id, + githubWorkflowRunId: 101n, + runAttempt: 3, + completedAt, + }) + + for (const runAttempt of [1, 3]) { + await recordGitHubBuildRunCompletion(prisma, { + buildId: build.id, + run: { runId: 101n, runAttempt }, + completedAt: new Date("2026-11-18T13:00:00Z"), + }) + } + + expect( + await prisma.gitHubBuildRunCompletion.findMany({ where: { buildId: build.id } }), + ).toEqual([newest]) + expect( + await prisma.build.findUniqueOrThrow({ + where: { id: build.id }, + include: { artifacts: true }, + }), + ).toEqual(build) + }) + + it.each(["running", "failed"] as const)( + "does not record a completed pass for a %s Build", + async (state) => { + const build = await createBuild(state) + + await expect( + recordGitHubBuildRunCompletion(prisma, { + buildId: build.id, + run: { runId: 100n, runAttempt: 2 }, + completedAt: new Date(), + }), + ).resolves.toBe(false) + + expect(await prisma.gitHubBuildRunCompletion.count()).toBe(0) + expect( + await prisma.build.findUniqueOrThrow({ + where: { id: build.id }, + include: { artifacts: true }, + }), + ).toEqual(build) + }, + ) +}) diff --git a/hub/server/repositories/github-builds/record-github-build-run-completion/index.ts b/hub/server/repositories/github-builds/record-github-build-run-completion/index.ts index 88b206fad..b288c2355 100644 --- a/hub/server/repositories/github-builds/record-github-build-run-completion/index.ts +++ b/hub/server/repositories/github-builds/record-github-build-run-completion/index.ts @@ -2,22 +2,41 @@ import type { PrismaClient } from "../../../generated/prisma/client.ts" import type { GitHubBuildRun } from "../types.ts" /** - * Records that the owning run finished and every artifact pass has run. Recovery treats - * a successful Build without this time as still owing a screenshot pass. + * Records a completed artifact pass per contributing run without changing dist ownership. + * Owning-run completion also retains its existing Build watermark. */ export async function recordGitHubBuildRunCompletion( database: PrismaClient, input: { buildId: string; run: Pick; completedAt: Date }, ) { - const result = await database.build.updateMany({ - where: { - id: input.buildId, - state: "successful", + return database.$transaction(async (transaction) => { + const build = await transaction.build.findFirst({ + where: { id: input.buildId, state: "successful" }, + select: { id: true }, + }) + if (!build) return false + const run = { + buildId: build.id, githubWorkflowRunId: input.run.runId, - githubWorkflowRunAttempt: input.run.runAttempt, - workflowCompletedAt: null, - }, - data: { workflowCompletedAt: input.completedAt }, + } + await transaction.gitHubBuildRunCompletion.createMany({ + data: [{ ...run, runAttempt: input.run.runAttempt, completedAt: input.completedAt }], + skipDuplicates: true, + }) + await transaction.gitHubBuildRunCompletion.updateMany({ + where: { ...run, runAttempt: { lt: input.run.runAttempt } }, + data: { runAttempt: input.run.runAttempt, completedAt: input.completedAt }, + }) + await transaction.build.updateMany({ + where: { + id: input.buildId, + state: "successful", + githubWorkflowRunId: input.run.runId, + githubWorkflowRunAttempt: input.run.runAttempt, + workflowCompletedAt: null, + }, + data: { workflowCompletedAt: input.completedAt }, + }) + return true }) - return result.count === 1 } diff --git a/hub/server/repositories/github-builds/settle-deselected-github-build/index.ts b/hub/server/repositories/github-builds/settle-deselected-github-build/index.ts new file mode 100644 index 000000000..0b6d73f90 --- /dev/null +++ b/hub/server/repositories/github-builds/settle-deselected-github-build/index.ts @@ -0,0 +1,51 @@ +import type { PrismaClient } from "../../../generated/prisma/client.ts" + +/** Stops only the unfinished Build still owned by the deselected run and attempt. */ +export async function settleDeselectedGitHubBuild( + database: PrismaClient, + input: { + githubRepositoryId: number + run: { + id: number + runAttempt: number + headBranch: string + headSha: string + status: string + updatedAt: Date + } + }, +) { + const { run } = input + const owned = { + githubWorkflowRunId: BigInt(run.id), + githubWorkflowRunAttempt: { lte: run.runAttempt }, + commitSha: run.headSha, + state: { in: ["queued" as const, "running" as const] }, + prototype: { + branchName: run.headBranch, + branchDeletedAt: null, + repository: { githubRepositoryId: BigInt(input.githubRepositoryId), removedAt: null }, + }, + } + return database.$transaction(async (transaction) => { + const build = await transaction.build.findFirst({ + where: owned, + select: { id: true, startedAt: true }, + }) + if (!build) return false + const finishedAt = new Date() + const result = await transaction.build.updateMany({ + where: { ...owned, id: build.id }, + data: { + state: "failed", + githubWorkflowRunAttempt: run.runAttempt, + startedAt: build.startedAt ?? finishedAt, + finishedAt, + workflowCompletedAt: run.status === "completed" ? run.updatedAt : null, + failureSummary: + "The Build workflow is no longer selected by builds.workflows in .github/replica.json on the repository's default branch. Select it again and rerun the workflow to retry.", + }, + }) + return result.count === 1 + }) +} diff --git a/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.db.test.ts b/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.db.test.ts index 0cf7da3fc..0a336f668 100644 --- a/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.db.test.ts +++ b/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.db.test.ts @@ -7,6 +7,7 @@ import { restoreHubSession } from "../../../../../domains/admission/restore-hub- import { runAdmissionTransaction } from "../../../../../domains/admission/run-admission-transaction" import { runSessionRestoreTransaction } from "../../../../../domains/admission/run-session-restore-transaction" import { syncGitHubInstallationRepositories as syncBoundGitHubInstallationRepositories } from "../../../../../domains/admission/sync-github-installation-repositories" +import type { GitHubWorkflowRunJob } from "../../../../../domains/github-builds/get-github-workflow-run-job-definition" import { createGitHubUserAuthenticationAdapter } from "../../../../../integrations/github/create-user-authentication-adapter" import { listInstallationRepositories } from "../../../../../integrations/github/list-installation-repositories" import { prisma } from "../../../../../integrations/prisma" @@ -1047,12 +1048,20 @@ describe("createGitHubWebhookRouteDependencies()", () => { expect(await prisma.gitHubWebhookDelivery.count()).toBe(1) }) - it("queues a Replica workflow import before recording the delivery", async () => { + it("queues workflow runs and completed jobs without reading GitHub or workflow selection", async () => { await createRepository(1001, "workflows") - const enqueueWorkflowRunImport = vi.fn(() => Promise.resolve()) + const request = vi + .spyOn(globalThis, "fetch") + .mockRejectedValue(new Error("Webhook intake must not contact GitHub.")) + const enqueueWorkflowRunImport = vi.fn( + async (_job: GitHubWorkflowRunJob, deliveryId: string) => { + expect( + await prisma.gitHubWebhookDelivery.findUnique({ where: { id: deliveryId } }), + ).toBeNull() + }, + ) const dependencies = createGitHubWebhookRouteDependencies({ enqueueWorkflowRunImport, - readBuildWorkflowPaths: () => Promise.resolve([".github/workflows/replica.yml"]), installationId: "700", provider: githubProvider, syncRepositories: () => Promise.resolve(), @@ -1076,6 +1085,7 @@ describe("createGitHubWebhookRouteDependencies()", () => { } const runDelivery = deliveryFor(runPayload) const jobDelivery = deliveryFor(jobPayload) + const otherDelivery = deliveryFor(otherWorkflow) await expect(dependencies.handleEvent("workflow_run", runPayload, runDelivery)).resolves.toBe( true, @@ -1084,154 +1094,70 @@ describe("createGitHubWebhookRouteDependencies()", () => { true, ) await expect( - dependencies.handleEvent("workflow_run", otherWorkflow, deliveryFor(otherWorkflow)), - ).resolves.toBe(false) + dependencies.handleEvent("workflow_run", otherWorkflow, otherDelivery), + ).resolves.toBe(true) expect(enqueueWorkflowRunImport.mock.calls).toEqual([ [{ githubRepositoryId: 1001, runId: 55 }, runDelivery.id], [{ githubRepositoryId: 1001, runId: 55 }, jobDelivery.id], + [{ githubRepositoryId: 1001, runId: 56 }, otherDelivery.id], ]) - expect(await prisma.gitHubWebhookDelivery.count()).toBe(2) - }) - - it("leaves a workflow delivery unrecorded when queueing fails so GitHub can redeliver it", async () => { - await createRepository(1001, "workflows") - const dependencies = createGitHubWebhookRouteDependencies({ - enqueueWorkflowRunImport: () => Promise.reject(new Error("Queue unavailable.")), - readBuildWorkflowPaths: () => Promise.resolve([".github/workflows/replica.yml"]), - installationId: "700", - provider: githubProvider, - syncRepositories: () => Promise.resolve(), - webhookSecret: "webhook-secret-for-tests", - }) - const payload = { - action: "completed", - installation: { id: 700 }, - repository: { id: 1001 }, - workflow_run: { id: 55, path: ".github/workflows/replica.yml" }, - } - - await expect( - dependencies.handleEvent("workflow_run", payload, deliveryFor(payload)), - ).rejects.toThrow("Queue unavailable.") - expect(await prisma.gitHubWebhookDelivery.count()).toBe(0) - }) - - it("queues only selected workflow callers while leaving completed jobs for the worker to verify", async () => { - await createRepository(1001, "custom-workflows") - const readBuildWorkflowPaths = vi.fn(() => - Promise.resolve([".github/workflows/publish-proto.yml", ".github/workflows/preview.yaml"]), - ) - const enqueueWorkflowRunImport = vi.fn(() => Promise.resolve()) - const dependencies = createGitHubWebhookRouteDependencies({ - installationId: "700", - provider: githubProvider, - syncRepositories: () => Promise.resolve(), - webhookSecret: "webhook-secret-for-tests", - readBuildWorkflowPaths, - enqueueWorkflowRunImport, - }) - for (const [index, path] of [ - ".github/workflows/publish-proto.yml@refs/heads/proto/custom", - ".github/workflows/preview.yaml", - ".github/workflows/replica.yml", - ".github/workflows/publish-proto.yml-extra", - ".github/workflows/nested/publish-proto.yml", - ].entries()) { - const payload = { - action: "completed", - repository: { id: 1001 }, - workflow_run: { id: 50 + index, path }, - } - await expect( - dependencies.handleEvent("workflow_run", payload, deliveryFor(payload)), - ).resolves.toBe(index < 2) - } - const readsBeforeJob = readBuildWorkflowPaths.mock.calls.length - const job = { - action: "completed", - repository: { id: 1001 }, - workflow_job: { run_id: 99 }, - } - await expect(dependencies.handleEvent("workflow_job", job, deliveryFor(job))).resolves.toBe( - true, - ) - - for (const [index, runId] of [50, 51, 99].entries()) { - expect(enqueueWorkflowRunImport).toHaveBeenNthCalledWith( - index + 1, - { githubRepositoryId: 1001, runId }, - expect.any(String), - ) - } - expect(readBuildWorkflowPaths).toHaveBeenCalledWith(1001) - expect(readBuildWorkflowPaths).toHaveBeenCalledTimes(readsBeforeJob) expect(await prisma.gitHubWebhookDelivery.count()).toBe(3) + expect(request).not.toHaveBeenCalled() }) - it.each(["unknown", "disconnected"])( - "does not read configuration for an %s repository's workflow", - async (state) => { - if (state === "disconnected") { - const repository = await createRepository(1001, "removed-workflows") - await prisma.repository.update({ - where: { id: repository.id }, - data: { removedAt: new Date() }, - }) - } - const readBuildWorkflowPaths = vi.fn(() => - Promise.reject(new Error("Must not contact an unconnected repository.")), - ) - const enqueueWorkflowRunImport = vi.fn(() => Promise.resolve()) + it.each(["workflow_run", "workflow_job"])( + "leaves a failed %s enqueue unrecorded so the same delivery can retry", + async (event) => { + await createRepository(1001, "retry-workflows") + const enqueueWorkflowRunImport = vi + .fn() + .mockRejectedValueOnce(new Error("Queue unavailable.")) + .mockResolvedValue(undefined) const dependencies = createGitHubWebhookRouteDependencies({ + enqueueWorkflowRunImport, installationId: "700", provider: githubProvider, syncRepositories: () => Promise.resolve(), webhookSecret: "webhook-secret-for-tests", - readBuildWorkflowPaths, - enqueueWorkflowRunImport, }) const payload = { action: "completed", repository: { id: 1001 }, - workflow_run: { id: 55, path: ".github/workflows/replica.yml" }, + ...(event === "workflow_run" + ? { workflow_run: { id: 55, path: ".github/workflows/publish.yml" } } + : { workflow_job: { run_id: 55 } }), } + const delivery = deliveryFor(payload) - await expect( - dependencies.handleEvent("workflow_run", payload, deliveryFor(payload)), - ).resolves.toBe(false) - expect(readBuildWorkflowPaths).not.toHaveBeenCalled() - expect(enqueueWorkflowRunImport).not.toHaveBeenCalled() + await expect(dependencies.handleEvent(event, payload, delivery)).rejects.toThrow( + "Queue unavailable.", + ) expect(await prisma.gitHubWebhookDelivery.count()).toBe(0) + + await expect(dependencies.handleEvent(event, payload, delivery)).resolves.toBe(true) + expect(enqueueWorkflowRunImport).toHaveBeenLastCalledWith( + { githubRepositoryId: 1001, runId: 55 }, + delivery.id, + ) + expect(await prisma.gitHubWebhookDelivery.count()).toBe(1) + expect(enqueueWorkflowRunImport).toHaveBeenCalledTimes(2) }, ) - it("fails clearly when workflow selection is unwired instead of silently ignoring a valid run", async () => { - await createRepository(1001, "unwired-workflows") - const enqueueWorkflowRunImport = vi.fn(() => Promise.resolve()) - const dependencies = createGitHubWebhookRouteDependencies({ - installationId: "700", - provider: githubProvider, - syncRepositories: () => Promise.resolve(), - webhookSecret: "webhook-secret-for-tests", - enqueueWorkflowRunImport, - }) - const payload = { - action: "completed", - repository: { id: 1001 }, - workflow_run: { id: 55, path: ".github/workflows/replica.yml" }, + it.each([ + { state: "unknown", event: "workflow_run" }, + { state: "disconnected", event: "workflow_run" }, + { state: "unknown", event: "workflow_job" }, + { state: "disconnected", event: "workflow_job" }, + ])("does not queue an $state repository's $event", async ({ state, event }) => { + if (state === "disconnected") { + const repository = await createRepository(1001, "removed-workflows") + await prisma.repository.update({ + where: { id: repository.id }, + data: { removedAt: new Date() }, + }) } - - await expect( - dependencies.handleEvent("workflow_run", payload, deliveryFor(payload)), - ).rejects.toThrow(/workflow.*(selection|configur|reader)|readBuildWorkflowPaths/i) - expect(enqueueWorkflowRunImport).not.toHaveBeenCalled() - expect(await prisma.gitHubWebhookDelivery.count()).toBe(0) - }) - - it("leaves configuration failures visible and unrecorded for later redelivery", async () => { - await createRepository(1001, "invalid-workflows") - let unavailable = true const enqueueWorkflowRunImport = vi.fn(() => Promise.resolve()) const dependencies = createGitHubWebhookRouteDependencies({ installationId: "700", @@ -1239,28 +1165,20 @@ describe("createGitHubWebhookRouteDependencies()", () => { syncRepositories: () => Promise.resolve(), webhookSecret: "webhook-secret-for-tests", enqueueWorkflowRunImport, - readBuildWorkflowPaths: () => - unavailable - ? Promise.reject(new Error(".github/replica.json builds.workflows is invalid.")) - : Promise.resolve([".github/workflows/publish.yml"]), }) const payload = { action: "completed", repository: { id: 1001 }, - workflow_run: { id: 55, path: ".github/workflows/publish.yml" }, + ...(event === "workflow_run" + ? { workflow_run: { id: 55, path: ".github/workflows/replica.yml" } } + : { workflow_job: { run_id: 55 } }), } - const delivery = deliveryFor(payload) - await expect(dependencies.handleEvent("workflow_run", payload, delivery)).rejects.toThrow( - "builds.workflows", + await expect(dependencies.handleEvent(event, payload, deliveryFor(payload))).resolves.toBe( + false, ) expect(enqueueWorkflowRunImport).not.toHaveBeenCalled() expect(await prisma.gitHubWebhookDelivery.count()).toBe(0) - - unavailable = false - await expect(dependencies.handleEvent("workflow_run", payload, delivery)).resolves.toBe(true) - expect(enqueueWorkflowRunImport).toHaveBeenCalledOnce() - expect(await prisma.gitHubWebhookDelivery.count()).toBe(1) }) it("applies simultaneous copies of one delivery only once", async () => { diff --git a/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.ts b/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.ts index a9268d138..1bd4b979e 100644 --- a/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.ts +++ b/hub/server/routes/api/github/webhooks/create-github-webhook-route-dependencies/index.ts @@ -81,7 +81,6 @@ export function createGitHubWebhookRouteDependencies(input: { syncRepositories: () => Promise /** Enqueues idempotently by key, so a redelivered event cannot import twice. */ enqueueWorkflowRunImport?: (job: GitHubWorkflowRunJob, key: string) => Promise - readBuildWorkflowPaths?: (githubRepositoryId: number) => Promise }): GitHubWebhookRouteDependencies { async function syncRepositories() { await input.syncRepositories() @@ -140,15 +139,6 @@ export function createGitHubWebhookRouteDependencies(input: { if (name === "workflow_run") { const event = workflowRunEvent.safeParse(payload) if (!event.success) return false - const githubRepositoryId = event.data.repository.id - if (!(await findConnectedGitHubRepository(prisma, githubRepositoryId))) return false - if (!input.readBuildWorkflowPaths) { - throw new Error("GitHub Build workflow configuration reader is required.") - } - const paths = await input.readBuildWorkflowPaths(githubRepositoryId) - if (!paths.includes(event.data.workflow_run.path.split("@")[0] ?? "")) { - return false - } job = { githubRepositoryId: event.data.repository.id, runId: event.data.workflow_run.id } } else { // Job events let the dist import as soon as its job ends, before screenshots finish. @@ -159,6 +149,8 @@ export function createGitHubWebhookRouteDependencies(input: { runId: event.data.workflow_job.run_id, } } + if (!(await findConnectedGitHubRepository(prisma, job.githubRepositoryId))) return false + // The worker reads GitHub and selection; intake must acknowledge without provider calls. // Enqueue before recording receipt so a failed enqueue can be redelivered. await input.enqueueWorkflowRunImport(job, delivery.id) await claimGitHubWebhookDelivery(prisma, receipt) diff --git a/hub/test/database/github-build-run-completion-migration/index.db.test.ts b/hub/test/database/github-build-run-completion-migration/index.db.test.ts new file mode 100644 index 000000000..4b3812276 --- /dev/null +++ b/hub/test/database/github-build-run-completion-migration/index.db.test.ts @@ -0,0 +1,196 @@ +import { execFile } from "node:child_process" +import { randomBytes, randomUUID } from "node:crypto" +import { readFile, readdir } from "node:fs/promises" +import { join } from "node:path" +import { fileURLToPath } from "node:url" +import { promisify } from "node:util" + +import pg from "pg" +import { describe, expect, it } from "vitest" + +const migrationTestTimeoutMs = 120_000 +// Last migration at the exact base, 1cf8ab2c63e931ca8034f7e4fd637a06e281f7bc. +const baseMigration = "20261217120000_github_recovery_ordering" + +describe("GitHub Build run completion migration", () => { + it( + "preserves the owning run and immutable artifacts while allowing per-run completion receipts", + async () => { + const adminUrl = process.env.TEST_DATABASE_URL + if (!adminUrl) throw new Error("TEST_DATABASE_URL is required.") + const name = `replica_run_completion_upgrade_${randomBytes(8).toString("hex")}` + const admin = new pg.Client({ connectionString: adminUrl }) + await admin.connect() + let created = false + try { + await admin.query(`CREATE DATABASE ${pg.escapeIdentifier(name)}`) + created = true + const databaseUrl = new URL(adminUrl) + databaseUrl.pathname = name + const database = new pg.Client({ connectionString: databaseUrl.toString() }) + try { + await database.connect() + const hubRoot = fileURLToPath(new URL("../../../", import.meta.url)) + const migrations = join(hubRoot, "schema/migrations") + const base = (await readdir(migrations, { withFileTypes: true })) + .filter((entry) => entry.isDirectory() && entry.name <= baseMigration) + .map(({ name }) => name) + .sort() + expect(base.at(-1)).toBe(baseMigration) + for (const migrationName of base) { + const sql = await readFile(join(migrations, migrationName, "migration.sql"), "utf8") + if (migrationName === "20261104190000_entra_oidc_authentication") { + const statements = sql.split("\n") + await database.query(statements.slice(0, 2).join("\n")) + await database.query(statements.slice(2).join("\n")) + } else { + await database.query(sql) + } + } + + const repositoryId = randomUUID() + const prototypeId = randomUUID() + const buildId = randomUUID() + const completedAt = new Date("2026-11-18T12:05:00Z") + await database.query( + `INSERT INTO repositories (id, github_repository_id, name_with_owner, default_branch) + VALUES ($1, 1001, 'team/design', 'main')`, + [repositoryId], + ) + await database.query( + `INSERT INTO prototypes + (id, repository_id, branch_name, name, registered_at, branch_head_sha, + current_build_run_created_at, current_build_run_id) + VALUES ($1, $2, 'proto/welcome', 'Welcome', '2026-11-18T12:01:00Z', $3, + '2026-11-18T12:00:00Z', 100)`, + [prototypeId, repositoryId, "a".repeat(40)], + ) + await database.query( + `INSERT INTO builds + (id, prototype_id, number, state, commit_sha, github_workflow_run_id, + github_workflow_run_attempt, workflow_url, workflow_created_at, + started_at, finished_at, workflow_completed_at, dist_digest) + VALUES ($1, $2, 1, 'successful', $3, 100, 2, + 'https://github.com/team/design/actions/runs/100', + '2026-11-18T12:00:00Z', '2026-11-18T12:00:00Z', + '2026-11-18T12:01:00Z', $4, $5)`, + [buildId, prototypeId, "a".repeat(40), completedAt, "sha256:" + "b".repeat(64)], + ) + await database.query("UPDATE prototypes SET current_build_id = $1 WHERE id = $2", [ + buildId, + prototypeId, + ]) + await database.query( + `INSERT INTO build_artifacts (id, build_id, kind, path, storage_key, byte_length) + VALUES ($1, $3, 'dist', 'dist/index.html', 'retained-dist', 20), + ($2, $3, 'screenshot', 'screenshots/desktop.png', 'retained-screenshot', 30)`, + [randomUUID(), randomUUID(), buildId], + ) + async function retainedBuild() { + const build = await database.query( + `SELECT repo.github_repository_id, prototype.id AS prototype_id, + prototype.name, prototype.registered_at, prototype.current_build_id, + prototype.branch_head_sha, prototype.current_build_run_created_at, + prototype.current_build_run_id, build.id AS build_id, build.state, + build.commit_sha, build.github_workflow_run_id, + build.github_workflow_run_attempt, build.workflow_url, + build.workflow_created_at, build.started_at, build.finished_at, + build.workflow_completed_at, build.dist_digest + FROM builds build + JOIN prototypes prototype ON prototype.id = build.prototype_id + JOIN repositories repo ON repo.id = prototype.repository_id + WHERE build.id = $1`, + [buildId], + ) + const artifacts = await database.query( + `SELECT id, build_id, kind, path, storage_key, byte_length + FROM build_artifacts WHERE build_id = $1 ORDER BY path`, + [buildId], + ) + return { builds: build.rows, artifacts: artifacts.rows } + } + const before = await retainedBuild() + expect(before.builds).toHaveLength(1) + expect(before.artifacts).toHaveLength(2) + + const run = promisify(execFile) + const prismaCli = join(hubRoot, "node_modules/prisma/build/index.js") + const options = { + cwd: hubRoot, + env: { ...process.env, DATABASE_URL: databaseUrl.toString() }, + timeout: migrationTestTimeoutMs, + } + for (const migrationName of base) { + await run( + process.execPath, + [ + prismaCli, + "migrate", + "resolve", + "--applied", + migrationName, + "--config", + "prisma7.config.ts", + ], + options, + ) + } + await run( + process.execPath, + [prismaCli, "migrate", "deploy", "--config", "prisma7.config.ts"], + options, + ) + + expect(await retainedBuild()).toEqual(before) + expect( + (await database.query("SELECT id FROM github_build_run_completions")).rows, + ).toEqual([]) + const receiptId = randomUUID() + const insert = `INSERT INTO github_build_run_completions + (id, build_id, github_workflow_run_id, run_attempt, completed_at) + VALUES ($1, $2, 101, 1, $3)` + await database.query(insert, [receiptId, buildId, completedAt]) + await expect( + database.query(insert, [randomUUID(), buildId, completedAt]), + ).rejects.toMatchObject({ code: "23505" }) + await expect( + database.query(insert, [randomUUID(), randomUUID(), completedAt]), + ).rejects.toMatchObject({ code: "23503" }) + const receipts = await database.query( + `SELECT id, build_id, github_workflow_run_id, run_attempt, completed_at + FROM github_build_run_completions`, + ) + expect(receipts.rows).toEqual([ + { + id: receiptId, + build_id: buildId, + github_workflow_run_id: "101", + run_attempt: 1, + completed_at: completedAt, + }, + ]) + const otherBuildId = randomUUID() + await database.query( + `INSERT INTO builds (id, prototype_id, number, state) + VALUES ($1, $2, 2, 'successful')`, + [otherBuildId, prototypeId], + ) + await database.query(insert, [randomUUID(), otherBuildId, completedAt]) + await expect( + database.query("DELETE FROM builds WHERE id = $1", [otherBuildId]), + ).rejects.toMatchObject({ code: "23503" }) + expect(await retainedBuild()).toEqual(before) + } finally { + await database.end() + } + } finally { + try { + if (created) await admin.query(`DROP DATABASE ${pg.escapeIdentifier(name)} WITH (FORCE)`) + } finally { + await admin.end() + } + } + }, + migrationTestTimeoutMs, + ) +}) From a1b33a70f6555416710f5b1eeec095ca80eb8b38 Mon Sep 17 00:00:00 2001 From: Will Eastler <32074613+weastler@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:17:49 -0800 Subject: [PATCH 4/4] test(hub): support PostgreSQL restrict error variants Check the named foreign-key constraint and retained Build references instead of a PostgreSQL-version-specific SQLSTATE. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../index.db.test.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/hub/test/database/github-build-run-completion-migration/index.db.test.ts b/hub/test/database/github-build-run-completion-migration/index.db.test.ts index 4b3812276..38a5bbd05 100644 --- a/hub/test/database/github-build-run-completion-migration/index.db.test.ts +++ b/hub/test/database/github-build-run-completion-migration/index.db.test.ts @@ -178,7 +178,20 @@ describe("GitHub Build run completion migration", () => { await database.query(insert, [randomUUID(), otherBuildId, completedAt]) await expect( database.query("DELETE FROM builds WHERE id = $1", [otherBuildId]), - ).rejects.toMatchObject({ code: "23503" }) + ).rejects.toMatchObject({ + constraint: "github_build_run_completions_build_id_fkey", + }) + expect( + ( + await database.query( + `SELECT build.id, receipt.build_id + FROM builds build + JOIN github_build_run_completions receipt ON receipt.build_id = build.id + WHERE build.id = $1`, + [otherBuildId], + ) + ).rows, + ).toEqual([{ id: otherBuildId, build_id: otherBuildId }]) expect(await retainedBuild()).toEqual(before) } finally { await database.end()