Skip to content

Commit f72b07e

Browse files
Ben ThomasCopilot
authored andcommitted
feat(hub): add decision model learnings flow
Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 7eb5c00 commit f72b07e

25 files changed

Lines changed: 4030 additions & 14 deletions

File tree

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
import { describe, expect, it } from "vitest"
2+
3+
import {
4+
decisionModelConfigurationResponseSchema,
5+
decisionModelOverviewResponseSchema,
6+
decisionTrainingMutationResponseSchema,
7+
} from "."
8+
9+
const digest = `sha256:${"a".repeat(64)}`
10+
11+
function overview() {
12+
return {
13+
preflight: {
14+
state: "ready" as const,
15+
counts: {
16+
prototypes: 2,
17+
builds: 5,
18+
pages: 8,
19+
screenshots: 32,
20+
questions: 7,
21+
samples: 43,
22+
tasks: 43,
23+
train: 27,
24+
dev: 8,
25+
test: 8,
26+
follows: 24,
27+
breaks: 19,
28+
},
29+
warnings: [],
30+
reasons: [],
31+
digests: {
32+
export: digest,
33+
reviewSnapshot: digest,
34+
buildSet: digest,
35+
questionSet: digest,
36+
taskset: digest,
37+
},
38+
},
39+
configuration: {
40+
enabled: true,
41+
generation: 3,
42+
backend: "customer-runtime",
43+
maxUpdates: 8,
44+
maxSeconds: 300,
45+
monthlyBudgetCredits: "10.5",
46+
},
47+
jobs: [
48+
{
49+
id: "11111111-2222-4333-8444-555555555555",
50+
state: "succeeded" as const,
51+
tasksetDigest: digest,
52+
runtimeJobId: "runtime-job-1",
53+
candidateRuntimeId: "candidate-1",
54+
failureCode: null,
55+
createdAt: "2026-11-11T18:00:00Z",
56+
finishedAt: "2026-11-11T18:04:30Z",
57+
},
58+
],
59+
candidates: [
60+
{
61+
id: "22222222-3333-4444-8555-666666666666",
62+
runtimeCandidateId: "candidate-1",
63+
tasksetDigest: digest,
64+
productionEligible: true,
65+
baselineMeanReward: "0.61",
66+
candidateMeanReward: "0.79",
67+
meanRewardDelta: "0.18",
68+
baselineCalibrationError: "0.2",
69+
candidateCalibrationError: "0.17",
70+
calibrationErrorDelta: "-0.03",
71+
learningMetrics: [
72+
{
73+
learningId: "learning-1",
74+
baselineBalancedAccuracy: 0.5,
75+
candidateBalancedAccuracy: 0.75,
76+
delta: 0.25,
77+
direction: "gain" as const,
78+
},
79+
],
80+
durationSeconds: 270,
81+
usage: [
82+
{
83+
connector: "customer-gpu",
84+
kind: "training_credits",
85+
quantity: 3.5,
86+
unit: "credits",
87+
cost: 1.25,
88+
currency: "USD",
89+
source: "runtime",
90+
metadata: {},
91+
},
92+
],
93+
rejection: null,
94+
createdAt: "2026-11-11T18:04:30Z",
95+
},
96+
],
97+
promotion: {
98+
candidateId: "22222222-3333-4444-8555-666666666666",
99+
runtimeCandidateId: "candidate-1",
100+
generation: 2,
101+
operation: "promote" as const,
102+
rollbackAvailable: true,
103+
},
104+
promotionHistory: [],
105+
evaluations: [],
106+
deletions: [],
107+
}
108+
}
109+
110+
describe("Decision-model contracts", () => {
111+
it("accepts only the persisted control-plane overview fields", () => {
112+
const response = overview()
113+
114+
expect(decisionModelOverviewResponseSchema.parse(response)).toEqual(response)
115+
expect(
116+
decisionModelOverviewResponseSchema.safeParse({
117+
...response,
118+
candidates: response.candidates.map((candidate) => ({
119+
...candidate,
120+
inventedProof: true,
121+
})),
122+
}).success,
123+
).toBe(false)
124+
})
125+
126+
it("keeps repository training settings within the supported bounds", () => {
127+
const configuration = overview().configuration
128+
129+
expect(
130+
decisionModelConfigurationResponseSchema.parse({
131+
...configuration,
132+
maxUpdates: 1,
133+
maxSeconds: 1,
134+
monthlyBudgetCredits: "0",
135+
}),
136+
).toMatchObject({ maxUpdates: 1, maxSeconds: 1, monthlyBudgetCredits: "0" })
137+
expect(
138+
decisionModelConfigurationResponseSchema.parse({
139+
...configuration,
140+
maxUpdates: 100_000,
141+
maxSeconds: 86_400,
142+
monthlyBudgetCredits: "999.123456789",
143+
}),
144+
).toMatchObject({
145+
maxUpdates: 100_000,
146+
maxSeconds: 86_400,
147+
monthlyBudgetCredits: "999.123456789",
148+
})
149+
150+
for (const invalid of [
151+
{ ...configuration, maxUpdates: 0 },
152+
{ ...configuration, maxUpdates: 100_001 },
153+
{ ...configuration, maxSeconds: 0 },
154+
{ ...configuration, maxSeconds: 86_401 },
155+
{ ...configuration, monthlyBudgetCredits: "-1" },
156+
{ ...configuration, monthlyBudgetCredits: "1.1234567890" },
157+
{ ...configuration, backend: "Fixture Backend" },
158+
]) {
159+
expect(decisionModelConfigurationResponseSchema.safeParse(invalid).success).toBe(false)
160+
}
161+
})
162+
163+
it("normalizes the server-shaped uppercase cancellation state", () => {
164+
expect(
165+
decisionTrainingMutationResponseSchema.parse({
166+
id: "11111111-2222-4333-8444-555555555555",
167+
state: "CANCELLED",
168+
repositoryId: "repository-1",
169+
}),
170+
).toMatchObject({
171+
id: "11111111-2222-4333-8444-555555555555",
172+
state: "cancelled",
173+
})
174+
})
175+
})

0 commit comments

Comments
 (0)