docs(benchmarks): link Hub-sourced review cards to their Harbor Hub trial pages

docs/benchmark_data/README.md

@@ -149,9 +149,13 @@ after review). The generator emits `safety.refused` / `safety.reference` / `safe
 (same inputs and options) and `moments.py` turns the export into `moments/<run>.json`:
 one card per pass with a high or critical finding and per review decision, with up to
 three findings and two masked excerpts each (said / ran / got at the matched step), led by
-the finding the decision cites. Cards link to the trial folder and the Hub's trace viewer
-in the public bucket (bucket runs only). `fetch_runs.py` records each task's trials in
-cell order (`trials`) for this. Excerpts are trace text: atif-scan masks secret shapes
+the finding the decision cites. Cards link to the trial's files and trajectory: the
+trial folder and the Hugging Face trace viewer for bucket runs, or the Harbor Hub trial
+page (it opens on the trajectory; files under `?tab=artifacts`) for Hub runs. Hub links
+need a public job: private jobs are recorded without one, and their cards stay unlinked.
+`fetch_runs.py` records each task's trials in cell order (`trials`, with each Hub trial's
+`job`) for this; `moments.py --relink` refreshes existing cards' links from the run files
+without rescanning. Excerpts are trace text: atif-scan masks secret shapes
 (best effort) and `moments.py` refuses token-shaped text; check the files against known
 credentials before committing. The generator copies them to `docs/benchmarks/moments/`;
 a run page loads its file when opened.

docs/benchmark_data/fetch_runs.py

@@ -904,6 +904,7 @@ def token_sum(key: str) -> int | None:
         vals = [t[key] for t in costed if t.get(key) is not None]
         return sum(vals) if vals else None
 
+    public_jobs = {j for j, show in shows.items() if show.get("visibility") == "public"}
     out: Json = {
         "id": run["id"],
         "benchmark": manifest["benchmark"],
@@ -940,6 +941,8 @@ def token_sum(key: str) -> int | None:
                     for k, v in (
                         ("name", t.get("trial_name") or t["id"]),
                         ("path", t.get("bucket_path")),
+                        # Hub trials of public jobs link to their Harbor Hub page.
+                        ("job", t.get("job_id") if t.get("job_id") in public_jobs else None),
                     )
                     if v
                 }

docs/benchmark_data/moments.py

@@ -8,13 +8,17 @@
 Output is ``moments/<run id>.json``: one card per trial worth showing, i.e. a reward with
 a high or critical finding, or any trial our review decided on, carrying the decision, the
 findings (our publish-side rules first, minus the checks they restate) and up to two
-moments each, plus links to the trial folder and trajectory in the published bucket.
+moments each, plus links to the trial's files and trajectory: in the published bucket, or
+on the Harbor Hub trial page for Hub-sourced runs.
 
 Excerpts are trace text. atif-scan masks secret shapes (best effort); this script also
 refuses to write token-shaped text. Check the output against known credentials before
 committing it (bench-run: ``release.known_credentials()``). Standard library only.
 
     python docs/benchmark_data/moments.py --run opus55-high-6h --highlights /tmp/hl/data.js
+
+``--relink`` refreshes only the cards' links from the run files (no scan needed), for one
+``--run`` or every moments file.
 """
 
 from __future__ import annotations
@@ -33,6 +37,7 @@
 MAX_FINDINGS = 3  # per card
 MAX_MOMENTS = 2  # per finding
 CLIP = 600  # characters per excerpt part
+HUB_JOBS = "https://hub.harborframework.com/jobs/"
 TOKEN = re.compile(
     r"\b(?:sk-[A-Za-z0-9_\-]{20,}|sk-ant-[A-Za-z0-9_\-]{20,}|hf_[A-Za-z0-9]{24,}|gh[oupsr]_[A-Za-z0-9]{20,}"
     r"|github_pat_[A-Za-z0-9_]{20,}|xai-[A-Za-z0-9]{40,}|AKIA[0-9A-Z]{16}|eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{10,})"
@@ -64,6 +69,18 @@ def _clip(text: str, keep: str) -> str:
     return "…" + text[-CLIP:] if keep == "end" else text[:CLIP] + "…"
 
 
+def trial_links(record: Json, bucket_repo: str) -> tuple[str | None, str | None]:
+    """(files, trajectory) links for a run file's trial record. A published trial links
+    into the bucket; a Hub trial to its Harbor Hub page, which opens on the trajectory."""
+    if path := record.get("path"):
+        folder = f"https://huggingface.co/buckets/{bucket_repo}/tree/{path}"
+        return folder, f"{folder}/agent/trajectory.json"  # the Hub's trace viewer
+    if job := record.get("job"):
+        page = f"{HUB_JOBS}{job}/trials/{record['name']}"
+        return f"{page}?tab=artifacts", page
+    return None, None
+
+
 def build(
     run_id: str,
     highlights: Json,
@@ -81,11 +98,11 @@ def build(
     rows = {r["key"]: r for g in highlights["groups"] for r in g["rows"]}
     restated = set(_restates().values())
 
-    # Trial name -> (task, attempt index in the task's cells, bucket path).
-    where: dict[str, tuple[str, int, str | None]] = {}
+    # Trial name -> (task, attempt index in the task's cells, run file trial record).
+    where: dict[str, tuple[str, int, Json]] = {}
     for task, trials in run["trials"].items():
         for i, t in enumerate(trials):
-            where[t["name"]] = (task, i, t.get("path"))
+            where[t["name"]] = (task, i, t)
 
     review = run.get("review") or {}
     decisions: dict[str, Json] = {}
@@ -161,22 +178,17 @@ def build(
         top = PRIORITY.get(findings[0]["priority"], 9) if findings else 9
         if name not in decisions and not (rewarded and top <= PRIORITY["high"]):
             continue
-        task, attempt, path = where[name]
+        task, attempt, record = where[name]
+        folder, trace = trial_links(record, bucket_repo)
         cards.append(
             {
                 "trial": label.get(name, name),
                 "task": task,
                 "attempt": attempt,
                 "reward": rewards.get(name),
                 **decisions.get(name, {"decision": None, "reason": None}),
-                "folder": f"https://huggingface.co/buckets/{bucket_repo}/tree/{path}"
-                if path
-                else None,
-                "trace": (  # the Hub's trace viewer for the trajectory
-                    f"https://huggingface.co/buckets/{bucket_repo}/tree/{path}/agent/trajectory.json"
-                    if path
-                    else None
-                ),
+                "folder": folder,
+                "trace": trace,
                 "findings": findings[:MAX_FINDINGS],
             }
         )
@@ -210,19 +222,44 @@ def check_clean(doc: Json) -> None:
         )
 
 
+def relink(run_id: str, bucket_repo: str) -> int:
+    """Refresh an existing moments file's links from the run file; returns cards linked."""
+    path = HERE / "moments" / f"{run_id}.json"
+    doc = json.loads(path.read_text())
+    run_file = next(p for p in (HERE / "runs", HERE / "tb4" / "runs") if (p / path.name).exists())
+    run = json.loads((run_file / path.name).read_text())
+    for card in doc["cards"]:
+        record = run["trials"][card["task"]][card["attempt"]]
+        card["folder"], card["trace"] = trial_links(record, bucket_repo)
+    path.write_text(json.dumps(doc, indent=1, ensure_ascii=False) + "\n", encoding="utf-8")
+    return sum(bool(card["trace"]) for card in doc["cards"])
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(
         description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
     )
-    parser.add_argument("--run", required=True)
+    parser.add_argument("--run", help="run id (all moments files with --relink)")
+    parser.add_argument("--highlights", type=Path, help="data.js from atif-scan --highlights")
     parser.add_argument(
-        "--highlights", type=Path, required=True, help="data.js from atif-scan --highlights"
+        "--relink", action="store_true", help="refresh existing cards' links from run files"
     )
     args = parser.parse_args()
 
-    manifest = json.loads((HERE / "manifest.json").read_text())
-    spec = next(r for r in manifest["runs"] if r["id"] == args.run)
-    repo = (spec.get("bucket") or {}).get("repo", "evalstate/published-benchmarks")
+    repos = {
+        r["id"]: (r.get("bucket") or {}).get("repo", "evalstate/published-benchmarks")
+        for manifest in (HERE / "manifest.json", HERE / "tb4" / "manifest.json")
+        for r in json.loads(manifest.read_text())["runs"]
+    }
+    if args.relink:
+        runs = [args.run] if args.run else sorted(p.stem for p in (HERE / "moments").glob("*.json"))
+        for run_id in runs:
+            doc = json.loads((HERE / "moments" / f"{run_id}.json").read_text())
+            print(f"{run_id}: {relink(run_id, repos[run_id])}/{len(doc['cards'])} cards linked")
+        return
+    if not (args.run and args.highlights):
+        parser.error("--run and --highlights are required (or use --relink)")
+    repo = repos[args.run]
     doc = build(args.run, load_highlights(args.highlights), repo)
     check_clean(doc)
     out = HERE / "moments" / f"{args.run}.json"

docs/benchmark_data/moments/cc-fable5-xhigh.json

@@ -11,8 +11,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/11efb542-89de-4746-8227-b776c7841a96/trials/f7d12944-5016-43ec-9111-97c0e83aca2c?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/11efb542-89de-4746-8227-b776c7841a96/trials/f7d12944-5016-43ec-9111-97c0e83aca2c",
    "findings": [
     {
      "check": "lookup.benchmark_content_obtained",

docs/benchmark_data/moments/cc-sonnet5-high.json

@@ -11,8 +11,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/84ac1a9d-52a7-491b-85fb-dc323231b67f/trials/99494529-0aae-4ff3-bfcb-970c27d37bf8?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/84ac1a9d-52a7-491b-85fb-dc323231b67f/trials/99494529-0aae-4ff3-bfcb-970c27d37bf8",
    "findings": [
     {
      "check": "tb21.fix_ocaml_gc.upstream_source",
@@ -54,8 +54,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/84ac1a9d-52a7-491b-85fb-dc323231b67f/trials/97093d67-07db-4182-b478-6b30dfa0363c?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/84ac1a9d-52a7-491b-85fb-dc323231b67f/trials/97093d67-07db-4182-b478-6b30dfa0363c",
    "findings": [
     {
      "check": "tb21.fix_ocaml_gc.upstream_source",
@@ -97,8 +97,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/84ac1a9d-52a7-491b-85fb-dc323231b67f/trials/435894f4-133f-470d-bb53-ef08483535eb?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/84ac1a9d-52a7-491b-85fb-dc323231b67f/trials/435894f4-133f-470d-bb53-ef08483535eb",
    "findings": [
     {
      "check": "tb21.fix_ocaml_gc.upstream_source",
@@ -140,8 +140,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/84ac1a9d-52a7-491b-85fb-dc323231b67f/trials/36844ea2-9b81-4854-82f9-7b50910732a1?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/84ac1a9d-52a7-491b-85fb-dc323231b67f/trials/36844ea2-9b81-4854-82f9-7b50910732a1",
    "findings": [
     {
      "check": "lookup.benchmark_lookup_answered",
@@ -221,8 +221,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/84ac1a9d-52a7-491b-85fb-dc323231b67f/trials/c0aeb474-34e6-4bbe-b952-0e6cd5ddb333?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/84ac1a9d-52a7-491b-85fb-dc323231b67f/trials/c0aeb474-34e6-4bbe-b952-0e6cd5ddb333",
    "findings": [
     {
      "check": "tb21.task_origin_solution",

docs/benchmark_data/moments/deepseek-v4-flash-vision-max-6h.json

@@ -11,8 +11,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/5cf4a449-18d4-4f14-80be-3f57581b7f16/trials/8bc4597a-8067-4d5e-b413-049333f6c3f5?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/5cf4a449-18d4-4f14-80be-3f57581b7f16/trials/8bc4597a-8067-4d5e-b413-049333f6c3f5",
    "findings": [
     {
      "check": "lookup.benchmark_content_obtained",
@@ -104,8 +104,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/5cf4a449-18d4-4f14-80be-3f57581b7f16/trials/24bee532-0edb-402f-a799-633fbd54f892?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/5cf4a449-18d4-4f14-80be-3f57581b7f16/trials/24bee532-0edb-402f-a799-633fbd54f892",
    "findings": [
     {
      "check": "lookup.benchmark_content_obtained",
@@ -197,8 +197,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/0f4579ec-3d0e-40a0-97b3-02209d12404d?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/0f4579ec-3d0e-40a0-97b3-02209d12404d",
    "findings": [
     {
      "check": "lookup.benchmark_content_obtained",
@@ -290,8 +290,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/67a5f998-a789-4ba9-8976-2137b1ee57e4?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/67a5f998-a789-4ba9-8976-2137b1ee57e4",
    "findings": [
     {
      "check": "tb21.fix_ocaml_gc.upstream_source",
@@ -333,8 +333,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/d12397c6-935a-4d85-8e5f-f150f14c8ba6?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/d12397c6-935a-4d85-8e5f-f150f14c8ba6",
    "findings": [
     {
      "check": "tb21.fix_ocaml_gc.upstream_source",
@@ -376,8 +376,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/ae1642aa-4da2-43fa-be64-adf9dbb122e0?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/ae1642aa-4da2-43fa-be64-adf9dbb122e0",
    "findings": [
     {
      "check": "tb21.fix_ocaml_gc.upstream_source",
@@ -419,8 +419,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/7661d572-4bfe-4e16-89ce-d4502fa0ba3e?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/7661d572-4bfe-4e16-89ce-d4502fa0ba3e",
    "findings": [
     {
      "check": "tb21.fix_ocaml_gc.upstream_source",
@@ -462,8 +462,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/5fead525-8852-4e37-b0ae-f682bd6e52ea?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/5fead525-8852-4e37-b0ae-f682bd6e52ea",
    "findings": [
     {
      "check": "tb21.fix_ocaml_gc.upstream_source",
@@ -505,8 +505,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/6171fc2d-e87f-4fc5-ba11-cf73fbcf808b?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/6171fc2d-e87f-4fc5-ba11-cf73fbcf808b",
    "findings": [
     {
      "check": "tb21.path_tracing_reverse.pixel_special_case",
@@ -536,8 +536,8 @@
    "reward": 1.0,
    "decision": null,
    "reason": null,
-   "folder": null,
-   "trace": null,
+   "folder": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/b931f713-37d6-486b-8ac4-b720a3b0a66b?tab=artifacts",
+   "trace": "https://hub.harborframework.com/jobs/852e75c9-ca0e-4551-9393-21835a9857e2/trials/b931f713-37d6-486b-8ac4-b720a3b0a66b",
    "findings": [
     {
      "check": "tb21.path_tracing_reverse.pixel_special_case",