battery: doc_check.py cross-checks the manifests against the dataset pages

open.json against docs/EXTERNAL_TEST_DATA.md (membership both ways, space group, cell, d_min,
the stated row and repository counts, the small-molecule references), still_manifest.json
against docs/EXTERNAL_TEST_DATA_STILLS.md (membership both ways, CXIDB entry, PDB entry, space
group, d_min) and inhouse.json against the README's Arms table; exits 1 on a mismatch.
Stdlib only, no data needed. The README names it beside test_score.py and in "Adding datasets".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015CeEGgpKdfkCTJb3YNMJcu

tools/battery/README.md

@@ -12,7 +12,11 @@ machine: the protocol, the REFMAC check, the timing benchmark, how to take timin
 what to send back.
 
 `test_score.py` covers the scoring rules that decide a verdict (stdlib `unittest`, no data and no
-run needed): `cd tools/battery && python3 test_score.py`.
+run needed): `cd tools/battery && python3 test_score.py`. `doc_check.py` checks the manifests
+against the pages that document their datasets - `open.json` against
+[External test data](../../docs/EXTERNAL_TEST_DATA.md), `still_manifest.json` against its stills
+page, `inhouse.json` against the Arms table below - and exits non-zero on a mismatch (stdlib, no
+data): `cd tools/battery && python3 doc_check.py`.
 
 ## Arms
 
@@ -832,7 +836,8 @@ whole range; the manifest keeps the rule used as `dmin_rule`.
   (the largest sweep, chosen by frame count), and with `--refresh` it shows where discovery now
   disagrees with an unpinned manifest input. Add the row to `open.json` with the deposited
   reference (`sg`, `sgno`, `cell`, `dmin`) and tags. Set `"pinned": true` when the choice between
-  sweeps was deliberate. Record the source and DOI in `docs/EXTERNAL_TEST_DATA.md`.
+  sweeps was deliberate. Record the source and DOI in `docs/EXTERNAL_TEST_DATA.md`, then run
+  `python3 doc_check.py`, which fails until the page and the manifests agree (below).
   A set whose frames are not the crystal the deposited data came from (checked independently,
   e.g. with XDS on the same frames) gets `"reference_mismatch": "<why>"`. It is an annotation only:
   every check still runs and every number still counts; the report prints the note beside the set's

tools/battery/doc_check.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""doc_check.py -- the battery manifests against the pages that document their datasets.
+
+  open.json            <-> docs/EXTERNAL_TEST_DATA.md
+  still_manifest.json  <-> docs/EXTERNAL_TEST_DATA_STILLS.md
+  inhouse.json         <-> the inhouse row of the Arms table in tools/battery/README.md
+
+Open arm: every manifest row is a row of the page's Datasets table (through the "Battery rows
+whose id is not the PDB code" table where the id is not the PDB code) or of "Datasets with no PDB
+entry"; every Datasets row is in the manifest or under "Datasets in the table that are not in the
+battery"; space group, cell (to the page's 0.1 A / 0.1 deg) and d_min (to 0.01 A) agree; the
+stated row count and the per-repository counts are the table's. Still battery: every set is in
+the page's table with its CXIDB entry, PDB entry, space group and d_min, and every set the page
+names is in the manifest. In-house: each set's specimen is named in the README.
+
+  python3 doc_check.py        (exit status 1 on any mismatch; stdlib only, no data needed)
+"""
+import collections
+import json
+import os
+import re
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+DOCS = os.path.join(HERE, "..", "..", "docs")
+# in-house id prefix -> the name the README's Arms table gives the specimen
+INHOUSE = {"lyso": "lysozyme", "lysoI": "lysozyme", "thau": "thaumatin", "insu": "insulin",
+           "cytc": "cytochrome C", "myob": "myoglobin", "nothing": "no-crystal controls",
+           "aspirin": "aspirin", "citricacid": "citric acid", "hepes": "HEPES", "yag": "YAG",
+           "kdp": "KDP", "lcystine": "L-cystine"}
+
+errors = []
+
+
+def section(text, heading):
+    """The body of a '## heading' section, up to the next '## '."""
+    body = text.split("\n## " + heading + "\n", 1)[1]
+    return body.split("\n## ", 1)[0]
+
+
+def table(body, n=0):
+    """Rows of the n-th Markdown table in body, as lists of stripped cells."""
+    blocks = re.findall(r"(?:^\|.*\n?)+", body, re.M)
+    rows = blocks[n].splitlines()[2:]
+    return [[c.strip() for c in r.strip("|").split(" | ")] for r in rows]
+
+
+def check_open():
+    sets = {s["id"]: s for s in json.load(open(os.path.join(HERE, "open.json")))["sets"]}
+    text = open(os.path.join(DOCS, "EXTERNAL_TEST_DATA.md")).read()
+    rows = {re.match(r"\[(\w{4})\]", r[0]).group(1): r for r in table(section(text, "Datasets"))}
+    renamed = {r[0].strip("`"): r[1] for r in table(section(text, "Battery rows whose id is not the PDB code"))}
+    absent = {r[0] for r in table(section(text, "Datasets in the table that are not in the battery"))}
+    no_pdb = {r[0].strip("`"): r for r in table(section(text, "Datasets with no PDB entry"))}
+    small = {r[0].strip("`"): r for r in table(section(text, "Datasets with no PDB entry"), 1)}
+
+    used = set()
+    for sid, s in sets.items():
+        if sid in no_pdb:
+            ref, row = s.get("ref"), small.get(sid)
+            if ref and row and (row[1].split("`")[1] != ref["sg"] or not same_cell(row[2].replace("/", " "), ref["cell"], 0.0005)):
+                errors.append(f"open {sid}: space group or cell differ from the small-molecule table")
+            continue
+        pdb = renamed.get(sid, sid.upper())
+        row = rows.get(pdb)
+        if row is None:
+            errors.append(f"open {sid}: not on EXTERNAL_TEST_DATA.md")
+            continue
+        used.add(pdb)
+        ref = s["ref"]
+        if row[4] != ref["sg"]:
+            errors.append(f"open {sid}: space group {row[4]!r} on the page, {ref['sg']!r} in the manifest")
+        if not same_cell(row[5], ref["cell"], 0.051):
+            errors.append(f"open {sid}: cell {row[5]} on the page, {ref['cell']} in the manifest")
+        if ref["dmin"] is not None and abs(float(row[3]) - ref["dmin"]) > 0.0051:
+            errors.append(f"open {sid}: d_min {row[3]} on the page, {ref['dmin']} in the manifest")
+    for pdb in sorted(set(rows) - used - absent):
+        errors.append(f"open {pdb}: on the page but not in open.json, nor listed as not in the battery")
+    for pdb in sorted(absent & used):
+        errors.append(f"open {pdb}: listed as not in the battery, but open.json has it")
+
+    m = re.search(r"The table holds (\d+) PDB entries", text)
+    if not m or int(m.group(1)) != len(rows):
+        errors.append(f"open: the page states {m and m.group(1)} PDB entries, its table has {len(rows)}")
+    repos = collections.Counter(r[1].split(" [")[0] for r in rows.values())
+    repos.update(r[1].split(" record")[0] for r in no_pdb.values())
+    stated = dict((name, int(n)) for name, n in re.findall(
+        r"([A-Za-z][\w ]*?) (\d+)[,.]", section(text, "The collection in numbers").split("- **Repository**")[1].split(":", 1)[1].split("\n- ")[0]))
+    if stated != dict(repos):
+        errors.append(f"open: repository counts on the page {stated}, the tables give {dict(repos)}")
+
+
+def same_cell(text, cell, tol):
+    vals = [float(v) for v in text.split()]
+    return len(vals) == 6 and all(abs(a - b) <= tol for a, b in zip(vals, cell))
+
+
+def check_still():
+    sets = {s["id"]: s for s in json.load(open(os.path.join(HERE, "still_manifest.json")))["sets"]}
+    text = open(os.path.join(DOCS, "EXTERNAL_TEST_DATA_STILLS.md")).read()
+    seen = set()
+    for row in table(section(text, "Datasets")):
+        ids = re.findall(r"`(\w+)`", row[0])
+        pdbs = re.findall(r"\[(\w{4})\]\(https://www.rcsb", row[4])
+        cxidb = int(re.match(r"\[(\d+)\]", row[1]).group(1))
+        sg, dmin = row[5].rsplit(", ", 1)
+        for i, sid in enumerate(ids):
+            seen.add(sid)
+            s = sets.get(sid)
+            if s is None:
+                errors.append(f"still {sid}: on the page but not in still_manifest.json")
+                continue
+            pdb = pdbs[i] if len(pdbs) == len(ids) else pdbs[0]
+            got = (s["cxidb"], s["pdb"], s["sg"], s["d_cut"])
+            if got != (cxidb, pdb, sg, float(dmin)):
+                errors.append(f"still {sid}: page {(cxidb, pdb, sg, float(dmin))}, manifest {got}")
+    for sid in sorted(set(sets) - seen):
+        errors.append(f"still {sid}: in still_manifest.json but not on the page")
+
+
+def check_inhouse():
+    sets = json.load(open(os.path.join(HERE, "inhouse.json")))["sets"]
+    readme = open(os.path.join(HERE, "README.md")).read()
+    arm = next(l for l in readme.splitlines() if l.startswith("| **inhouse** |"))
+    for s in sets:
+        name = INHOUSE.get(s["id"].split("_")[0])
+        if name is None or name not in arm:
+            errors.append(f"inhouse {s['id']}: its specimen is not named in README.md's Arms table")
+
+
+check_open()
+check_still()
+check_inhouse()
+for e in errors:
+    print(e)
+print("doc_check: " + (f"{len(errors)} mismatch(es)" if errors else "manifests and docs agree"))
+sys.exit(1 if errors else 0)