bench: e2e skill eval v2 on papers the model does not know

benchmarks/e2e/README.md

@@ -4,6 +4,9 @@ This folder measures what an agent actually reports when a researcher hands it a
 asks it to check the references, with and without the ref-verify skill. It is a measurement,
 not a test: nothing here runs in CI, and nothing here changes the engine or `SKILL.md`.
 
+`v2/` repeats it with obscure 2025–2026 papers the model does not know, and adds a condition
+without web tools or a shell.
+
 ## Set
 
 Eight short manuscripts in `manuscripts/`, 11–12 references each (92 references, 21 cited

benchmarks/e2e/engine_only.py

@@ -5,7 +5,7 @@
 the engine's verdict, so "what the engine alone would flag" can be compared with what the
 agent reported. Needs the network.
 
-Usage: python3 engine_only.py --scratch DIR
+Usage: python3 engine_only.py --scratch DIR [--truth v2/truth.json]
 """
 
 from __future__ import annotations
@@ -27,28 +27,59 @@ def reference_section(text: str) -> str:
     return text[match.end():] if match else text
 
 
+def pair_results(items: list[dict], results: list[dict]) -> list[tuple[dict, dict | None]]:
+    # v1 lists items in reference order; v2 does not. Pair by the DOI the reference cites first,
+    # then by the most identifying strings found in the result's key or raw text.
+    unused = list(results)
+    paired: dict[str, dict] = {}
+    for item in items:
+        cited = (item.get("cited_doi") or item.get("truth_doi") or "").casefold()
+        match = next((r for r in unused if cited and (r.get("doi") or "").casefold() == cited), None)
+        if match is not None:
+            unused.remove(match)
+            paired[item["id"]] = match
+    for item in items:
+        if item["id"] in paired:
+            continue
+
+        def hits(result: dict) -> int:
+            text = f"{result.get('key') or ''} {result['reference']['raw']}"
+            return sum(token in text for token in item["match"])
+
+        best = max(unused, key=hits, default=None)
+        if best is not None and hits(best):
+            unused.remove(best)
+            paired[item["id"]] = best
+    return [(item, paired.get(item["id"])) for item in items]
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--scratch", type=Path, required=True)
+    parser.add_argument("--truth", type=Path, default=HERE / "truth.json")
     args = parser.parse_args()
     args.scratch.mkdir(parents=True, exist_ok=True)
-    truth = json.loads((HERE / "truth.json").read_text(encoding="utf-8"))
+    truth = json.loads(args.truth.read_text(encoding="utf-8"))
+    base = args.truth.resolve().parent
     env = dict(os.environ, PYTHONPATH=str(REPO / "src"), REF_VERIFY_CACHE_DIR=str(args.scratch / "cache"))
     out = {}
     for name, manuscript in truth["manuscripts"].items():
         if manuscript.get("bib"):
-            source = HERE / manuscript["bib"]
+            source = base / manuscript["bib"]
         else:
             source = args.scratch / f"{name}.txt"
-            source.write_text(reference_section((HERE / manuscript["path"]).read_text(encoding="utf-8")), encoding="utf-8")
+            source.write_text(reference_section((base / manuscript["path"]).read_text(encoding="utf-8")), encoding="utf-8")
         process = subprocess.run(
             [sys.executable, "-m", "ref_verify.cli", "check-bib", str(source), "--json"],
             env=env, capture_output=True, text=True, check=False,
         )
         results = json.loads(process.stdout)["results"]
         out[name] = results
         reference_items = [item for item in manuscript["items"] if not item["type"].startswith("CLAIM_")]
-        for item, result in zip(reference_items, results):
+        for item, result in pair_results(reference_items, results):
+            if result is None:
+                print(f"{item['id']:22} {item['type']:24} {'NO MATCH':10}")
+                continue
             label = "UNVERIFIED" if result["status"] == "UNVERIFIED" else result["verdict"]
             print(f"{item['id']:22} {item['type']:24} {label:10} {result['error_code'] or '':22} {result['reason'][:90]}")
     (args.scratch / "engine_only.json").write_text(json.dumps(out, ensure_ascii=False, indent=1), encoding="utf-8")

benchmarks/e2e/run_e2e.py

@@ -2,6 +2,8 @@
 
 Condition A copies the skill (SKILL.md + src/, as `npx skills add` lays it out) into
 <scratch project>/.claude/skills/ref-verify/; condition B gets the same project without it.
+Condition C (v2) is B with only Read, Glob, and Grep allowed: no web tools and no shell, so the
+agent can only answer from what the model already knows.
 Every run uses a fresh scratch project and a fresh engine cache, so runs do not share state.
 
 Isolation of the child sessions (verified before the first real run):
@@ -13,7 +15,8 @@
   CLAUDE_REFVERIFY_EVAL_CHILD=1  marks the process as an eval child
 A run whose stream reports any hook event is stopped and the harness exits.
 
-Usage: python3 run_e2e.py --scratch DIR [--only m1_dea_en:A:1 ...] [--parallel 2]
+Usage: python3 run_e2e.py --scratch DIR [--truth v2/truth.json] [--conditions A B C]
+                         [--model claude-opus-5-5] [--only m1_dea_en:A:1 ...] [--parallel 2]
 """
 
 from __future__ import annotations
@@ -32,8 +35,11 @@
 REPO = HERE.parents[1]
 CLAUDE = os.environ.get("CLAUDE_BIN", str(Path.home() / ".local/bin/claude"))
 SETTINGS = json.dumps({"disableAllHooks": True, "autoMemoryEnabled": False})
-ALLOWED_TOOLS = "Bash Read Glob Grep WebFetch WebSearch Skill"
-DISALLOWED_TOOLS = "Write Edit NotebookEdit"
+TOOLS = {
+    "A": ("Bash Read Glob Grep WebFetch WebSearch Skill", "Write Edit NotebookEdit"),
+    "B": ("Bash Read Glob Grep WebFetch WebSearch Skill", "Write Edit NotebookEdit"),
+    "C": ("Read Glob Grep", "Bash WebFetch WebSearch Write Edit NotebookEdit Skill"),
+}
 TIMEOUT_SECONDS = 1800
 
 PROMPTS = {
@@ -43,7 +49,10 @@
 }
 
 
-def command(prompt: str) -> list[str]:
+def command(prompt: str, condition: str = "A", model: str | None = None) -> list[str]:
+    allowed, disallowed = TOOLS[condition]
+    # v2 pins the model: the subscription default moved from Opus to Sonnet between v1 and v2.
+    pinned = ["--model", model] if model else []
     return [
         CLAUDE,
         "-p",
@@ -63,20 +72,21 @@ def command(prompt: str) -> list[str]:
         "--permission-mode",
         "dontAsk",
         "--allowedTools",
-        ALLOWED_TOOLS,
+        allowed,
         "--disallowedTools",
-        DISALLOWED_TOOLS,
+        disallowed,
+        *pinned,
     ]
 
 
-def prepare_project(root: Path, manuscript: dict, with_skill: bool) -> list[str]:
+def prepare_project(root: Path, base: Path, manuscript: dict, with_skill: bool) -> list[str]:
     if root.exists():
         raise SystemExit(f"{root} already exists; use a fresh --scratch directory")
     root.mkdir(parents=True)
     files = []
     for key in ("path", "bib"):
         if manuscript.get(key):
-            source = HERE / manuscript[key]
+            source = base / manuscript[key]
             shutil.copy2(source, root / source.name)
             files.append(source.name)
     if with_skill:
@@ -97,10 +107,10 @@ def _events(transcript: Path) -> list[dict]:
     return events
 
 
-def run_one(scratch: Path, name: str, manuscript: dict, condition: str, repeat: int) -> dict:
+def run_one(scratch: Path, base: Path, model: str | None, name: str, manuscript: dict, condition: str, repeat: int) -> dict:
     run_id = f"{name}__{condition}__{repeat}"
     project = scratch / "projects" / run_id
-    files = prepare_project(project, manuscript, with_skill=condition == "A")
+    files = prepare_project(project, base, manuscript, with_skill=condition == "A")
     prompt = PROMPTS[manuscript["language"]].format(files=", ".join(files))
     transcript = scratch / "transcripts" / f"{run_id}.jsonl"
     transcript.parent.mkdir(parents=True, exist_ok=True)
@@ -109,7 +119,7 @@ def run_one(scratch: Path, name: str, manuscript: dict, condition: str, repeat:
     with transcript.open("w", encoding="utf-8") as out, open(os.devnull) as devnull:
         try:
             process = subprocess.run(
-                command(prompt),
+                command(prompt, condition, model),
                 cwd=project,
                 env=env,
                 stdin=devnull,
@@ -142,26 +152,31 @@ def run_one(scratch: Path, name: str, manuscript: dict, condition: str, repeat:
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--scratch", type=Path, required=True)
+    parser.add_argument("--truth", type=Path, default=HERE / "truth.json")
+    parser.add_argument("--conditions", nargs="+", default=["A", "B"], choices=sorted(TOOLS))
+    parser.add_argument("--model", help="pin the child model (v1 ran on the default, then claude-opus-5-5)")
     parser.add_argument("--only", nargs="*", help="run ids like m1_dea_en:A:1")
     parser.add_argument("--repeats", type=int, default=2)
     parser.add_argument("--parallel", type=int, default=1, choices=(1, 2))
     args = parser.parse_args()
 
-    truth = json.loads((HERE / "truth.json").read_text(encoding="utf-8"))
+    truth = json.loads(args.truth.read_text(encoding="utf-8"))
+    base = args.truth.resolve().parent
     jobs = [
         (name, manuscript, condition, repeat)
         for repeat in range(1, args.repeats + 1)
         for name, manuscript in truth["manuscripts"].items()
-        for condition in ("A", "B")
+        for condition in args.conditions
     ]
     if args.only:
         wanted = {tuple(item.split(":")) for item in args.only}
         jobs = [job for job in jobs if (job[0], job[2], str(job[3])) in wanted]
-    print("command template:", json.dumps(command("<PROMPT>")), flush=True)
+    for condition in args.conditions:
+        print(f"command template {condition}:", json.dumps(command("<PROMPT>", condition, args.model)), flush=True)
     log = args.scratch / "runs.jsonl"
     args.scratch.mkdir(parents=True, exist_ok=True)
     with ThreadPoolExecutor(max_workers=args.parallel) as executor:
-        for record in executor.map(lambda job: run_one(args.scratch, *job), jobs):
+        for record in executor.map(lambda job: run_one(args.scratch, base, args.model, *job), jobs):
             with log.open("a", encoding="utf-8") as handle:
                 handle.write(json.dumps(record, ensure_ascii=False) + "\n")
     return 0

benchmarks/e2e/score.py

@@ -8,7 +8,7 @@
 automatic ones. It also records whether the run invoked the ref-verify engine, used the web,
 and how long it took.
 
-Usage: python3 score.py --scratch DIR [--review] [--markdown]
+Usage: python3 score.py --scratch DIR [--truth v2/truth.json] [--review] [--markdown]
 """
 
 from __future__ import annotations
@@ -21,6 +21,7 @@
 from pathlib import Path
 
 HERE = Path(__file__).resolve().parent
+CONDITION_LABELS = {"A": "A: with ref-verify skill", "B": "B: without", "C": "C: without, no web or shell"}
 
 # Words that show which problem the answer names, per error type (English and Korean).
 PROBLEM_WORDS = {
@@ -121,9 +122,9 @@ def automatic_decision(item: dict, answer: str) -> str:
     return "caught" if any(word.casefold() in lowered for word in words) else "review"
 
 
-def score(scratch: Path) -> dict:
-    truth = json.loads((HERE / "truth.json").read_text(encoding="utf-8"))
-    adjudication_path = HERE / "adjudication.json"
+def score(scratch: Path, truth_path: Path) -> dict:
+    truth = json.loads(truth_path.read_text(encoding="utf-8"))
+    adjudication_path = truth_path.parent / "adjudication.json"
     adjudication = json.loads(adjudication_path.read_text(encoding="utf-8")) if adjudication_path.exists() else {}
     runs = [json.loads(line) for line in (scratch / "runs.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()]
     scored = []
@@ -158,7 +159,7 @@ def summarize(result: dict) -> dict:
     truth = result["truth"]
     items = {item["id"]: item for m in truth["manuscripts"].values() for item in m["items"]}
     summary = {}
-    for condition in ("A", "B"):
+    for condition in sorted({run["condition"] for run in result["runs"]}):
         runs = [run for run in result["runs"] if run["condition"] == condition]
         recall: dict[str, list[int]] = {}
         false_alarms = {"correct_reference": [0, 0], "unindexed_reference": [0, 0], "correct_claim": [0, 0]}
@@ -199,34 +200,38 @@ def markdown(summary: dict, truth: dict) -> str:
     def pct(pair: list[int]) -> str:
         return f"{pair[0]}/{pair[1]} ({100 * pair[0] / pair[1]:.0f}%)" if pair[1] else "-"
 
-    lines = ["| | A: with ref-verify skill | B: without |", "|---|---|---|"]
+    conditions = sorted(summary)
+    lines = ["| | " + " | ".join(CONDITION_LABELS[c] for c in conditions) + " |", "|---" * (len(conditions) + 1) + "|"]
+
+    def row(label: str, cells: list[str]) -> None:
+        lines.append(f"| {label} | " + " | ".join(cells) + " |")
+
     for kind in truth["error_types"]:
-        lines.append(f"| {kind} | {pct(summary['A']['recall_by_type'].get(kind, [0, 0]))} | "
-                     f"{pct(summary['B']['recall_by_type'].get(kind, [0, 0]))} |")
-    lines.append(f"| **All planted errors** | **{pct(summary['A']['recall_overall'])}** | **{pct(summary['B']['recall_overall'])}** |")
+        row(kind, [pct(summary[c]["recall_by_type"].get(kind, [0, 0])) for c in conditions])
+    row("**All planted errors**", [f"**{pct(summary[c]['recall_overall'])}**" for c in conditions])
     for bucket, label in (("correct_reference", "False alarms: correct references"),
                           ("unindexed_reference", "False alarms: unindexed thesis/KCI/arXiv/JMLR"),
                           ("correct_claim", "False alarms: correct numeric claims")):
-        lines.append(f"| {label} | {pct(summary['A']['false_alarms'][bucket])} | {pct(summary['B']['false_alarms'][bucket])} |")
+        row(label, [pct(summary[c]["false_alarms"][bucket]) for c in conditions])
     for key, label in (("engine_invoked", "Runs that ran the ref-verify engine"),
                        ("skill_loaded", "Runs that loaded the skill"),
                        ("web_tool_runs", "Runs that used WebFetch/WebSearch"),
                        ("usable", "Final answer usable (names reference, problem, fix)")):
-        lines.append(f"| {label} | {summary['A'][key]}/{summary['A']['runs']} | {summary['B'][key]}/{summary['B']['runs']} |")
-    lines.append(f"| Wall time per run, median (mean) | {summary['A']['wall_seconds_median']} s "
-                 f"({summary['A']['wall_seconds_mean']} s) | {summary['B']['wall_seconds_median']} s "
-                 f"({summary['B']['wall_seconds_mean']} s) |")
+        row(label, [f"{summary[c][key]}/{summary[c]['runs']}" for c in conditions])
+    row("Wall time per run, median (mean)",
+        [f"{summary[c]['wall_seconds_median']} s ({summary[c]['wall_seconds_mean']} s)" for c in conditions])
     return "\n".join(lines)
 
 
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--scratch", type=Path, required=True)
+    parser.add_argument("--truth", type=Path, default=HERE / "truth.json")
     parser.add_argument("--review", action="store_true", help="list the decisions a person still has to make")
     parser.add_argument("--markdown", action="store_true")
     parser.add_argument("--out", type=Path, help="write per-run results JSON here")
     args = parser.parse_args()
-    result = score(args.scratch)
+    result = score(args.scratch, args.truth)
     summary = summarize(result)
     if args.review:
         items = {item["id"]: item for m in result["truth"]["manuscripts"].values() for item in m["items"]}