From 52025483d83af9b392e048cac4d3551141de6aed Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Sat, 17 Oct 2026 12:21:34 +0200 Subject: [PATCH] battery: a per-set reference_mismatch key skips the REFMAC and depositor-data checks The frames of 3mc4 and 3r6o are a crystal twinned ~30% by k,h,-l - XDS on the same frames reads the same H-test (0.29/0.27) as rugnux (0.29/0.29), in every 60-frame wedge and in P1 - while the deposited structure factors read untwinned (H ~0). The deposited data are another crystal of the same form, so the REFMAC R-free ratio and dep_cc on these two sets measured the mismatch, not rugnux. open.json carries the reason as "reference_mismatch"; run and recheck then record refmac_status/dep_status "skipped" with it, and the report lists both skips (the depositor-data skip list is shown even when no set was compared). Space group, cell and d_min are still scored against the deposition. Checked: battery run --only 3mc4,3r6o --model-check --gpulock (20261017-1139_e25876_twin-isa-mismatch) - both sets skipped with the reason; test_score.py OK. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PXc1zD1yxHqaR1DGdCTHL4 --- tools/battery/README.md | 5 +++++ tools/battery/battery.py | 24 +++++++++++++++++++++--- tools/battery/open.json | 4 ++-- tools/battery/report.py | 9 ++++++--- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/tools/battery/README.md b/tools/battery/README.md index 6519ec491..5629ef4b1 100644 --- a/tools/battery/README.md +++ b/tools/battery/README.md @@ -859,6 +859,11 @@ whole range; the manifest keeps the rule used as `dmin_rule`. disagrees with an unpinned manifest input. Add the row to `open.json` with the deposited reference (`sg`, `sgno`, `cell`, `dmin`) and tags. Set `"pinned": true` when the choice between sweeps was deliberate. Record the source and DOI in `docs/EXTERNAL_TEST_DATA.md`. + A set whose frames are not the crystal the deposited data came from (checked independently, + e.g. with XDS on the same frames) gets `"reference_mismatch": ""`: the REFMAC check and the + depositor-data comparison are skipped (`refmac_status` / `dep_status` `skipped`, the reason in + `refmac_reason` / `dep_reason` and in the report), while space group, cell and d_min are still + scored against the deposition. - **Small molecules** carry `"cod": ""`, the published structure the SHELXL check refines (pick an entry whose Niggli-reduced cell and space group match the row's reference; prefer a modern, ordered, low-R one). diff --git a/tools/battery/battery.py b/tools/battery/battery.py index 0c1dc3934..a7df13e9f 100644 --- a/tools/battery/battery.py +++ b/tools/battery/battery.py @@ -477,7 +477,7 @@ def run_one(run_dir, binary, e, opts): if e.get("timing"): row.update(timing.check(e["timing"], row, wd, reference)) elif e["reference"] == "deposition" and row["model"] and os.path.exists(os.path.join(wd, "p.mtz")): - row.update(depdata_check.check_set(wd, e["id"], opts["pdb_cache"])) + row.update(check_depdata(e, wd, opts["pdb_cache"])) if opts["model_check"] and e["arm"] == "open": row.update(check_model(e, wd)) if e.get("cod"): @@ -496,6 +496,20 @@ REFMAC_KEYS = ("refmac_rfree", "refmac_rwork", "refmac_rfree_depflags", "refmac_ "refmac_reason", "refmac_refined_ratio") + tuple("refmac_" + k for k in REFMAC_REFINED) +def mismatch_reason(e): + """Why the deposited data are not a reference for this set's frames (open.json's + `reference_mismatch`), or None: the REFMAC and depositor-data checks are then skipped.""" + return e.get("reference_mismatch") and "reference mismatch: " + e["reference_mismatch"] + + +def check_depdata(e, wd, pdb_cache): + """Our merge against the depositor's data (depdata_check.py), unless the manifest says the + deposition is not a reference for these frames.""" + if mismatch_reason(e): + return dict({k: None for k in depdata_check.KEYS}, dep_status="skipped", dep_reason=mismatch_reason(e)) + return depdata_check.check_set(wd, e["id"], pdb_cache) + + def check_model(e, wd): """R-free of the deposited model against this run's merge, by REFMAC (model_check.py).""" out = {k: None for k in REFMAC_KEYS} @@ -503,6 +517,8 @@ def check_model(e, wd): mtz = os.path.join(wd, "p.mtz") if not PDB_ID.match(pdb): return dict(out, refmac_status="skipped", refmac_reason="not a PDB entry") + if mismatch_reason(e): + return dict(out, refmac_status="skipped", refmac_reason=mismatch_reason(e)) if not os.path.exists(mtz): return dict(out, refmac_status="skipped", refmac_reason="no merged MTZ") try: @@ -989,7 +1005,7 @@ def recheck_one(e, wd, pdb_cache): """The post-processing checks of one open-arm row, on the run's own p.mtz (a worker process).""" out = {} if e["model"] and os.path.exists(os.path.join(wd, "p.mtz")): - out.update(depdata_check.check_set(wd, e["id"], pdb_cache)) + out.update(check_depdata(e, wd, pdb_cache)) out.update(check_model(e, wd)) return out @@ -1031,8 +1047,10 @@ def cmd_recheck(a, site): set_writable([os.path.join(wd, "model_check")]) os.rename(os.path.join(wd, "model_check"), os.path.join(wd, "model_check.pre-recheck-" + stamp)) shutil.copy2(os.path.join(run, "results.json"), os.path.join(run, "results.json.pre-recheck-" + stamp)) + mismatch = {e["id"]: e.get("reference_mismatch") for e in load_sets(site, "open")} with concurrent.futures.ProcessPoolExecutor(a.jobs) as pool: - futures = {pool.submit(recheck_one, {"id": r["set"], "model": r.get("model")}, wd, pdb_cache): r + futures = {pool.submit(recheck_one, {"id": r["set"], "model": r.get("model"), + "reference_mismatch": mismatch.get(r["set"])}, wd, pdb_cache): r for r, wd in zip(rows, wds)} for n, fut in enumerate(concurrent.futures.as_completed(futures), 1): r = futures[fut] diff --git a/tools/battery/open.json b/tools/battery/open.json index 10ea5cf1d..d2686dfd3 100644 --- a/tools/battery/open.json +++ b/tools/battery/open.json @@ -3,10 +3,10 @@ {"id": "36gk", "input": "36gk/CLS-0074_5-3_36GK/data/CLS-0074_5-3_master.h5", "wavelength": 0.95371, "ref": {"sg": "I 2 2 2", "sgno": 23, "cell": [120.58, 189.49, 199.69, 90.0, 90.0, 90.0], "dmin": 2.28}, "tags": ["h5", "orthorhombic"]}, {"id": "3inp", "input": "3inp/IDP02542_3inp/data/idp02542b.001", "wavelength": 0.97872, "ref": {"sg": "F 41 3 2", "sgno": 210, "cell": [224.08, 224.08, 224.08, 90.0, 90.0, 90.0], "dmin": 2.05}, "tags": ["marCCD", "cubic"], "tiers": {"ci": "marCCD, face-centred cubic F4132"}}, {"id": "3ky7", "input": "3ky7/IDP90258_3ky7/data/idp90258f.001", "wavelength": 0.97856, "ref": {"sg": "P 43 3 2", "sgno": 212, "cell": [125.176, 125.176, 125.176, 90.0, 90.0, 90.0], "dmin": 2.35}, "tags": ["marCCD", "cubic"]}, - {"id": "3mc4", "input": "3mc4/series/206918e4_x0001.img", "wavelength": 1.54178, "ref": {"sg": "H 3", "sgno": 146, "cell": [104.03, 104.03, 105.54, 90.0, 90.0, 120.0], "dmin": 1.95}, "tags": ["smv", "trigonal", "home-source"]}, + {"id": "3mc4", "input": "3mc4/series/206918e4_x0001.img", "wavelength": 1.54178, "ref": {"sg": "H 3", "sgno": 146, "cell": [104.03, 104.03, 105.54, 90.0, 90.0, 120.0], "dmin": 1.95}, "reference_mismatch": "the frames are a crystal twinned ~30% (law k,h,-l, confirmed with XDS on the same frames); the deposited data are an untwinned crystal of the same form", "tags": ["smv", "trigonal", "home-source"]}, {"id": "3meb", "input": "3meb/series/202097g3_x0001.img", "wavelength": 1.54178, "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [58.58, 101.15, 81.53, 90.0, 90.62, 90.0], "dmin": 1.9}, "tags": ["smv", "monoclinic", "home-source"]}, {"id": "3p85", "input": "3p85/series/217175d10_x0001.img", "wavelength": 1.54178, "ref": {"sg": "P 63 2 2", "sgno": 182, "cell": [127.29, 127.29, 72.9, 90.0, 90.0, 120.0], "dmin": 1.9}, "tags": ["smv", "hexagonal", "home-source"]}, - {"id": "3r6o", "input": "3r6o/series/219594b8_x0001.img", "wavelength": 1.54178, "ref": {"sg": "I 41", "sgno": 80, "cell": [90.68, 90.68, 76.13, 90.0, 90.0, 90.0], "dmin": 1.95}, "symmetry_evidence": "symmetry/3r6o", "tags": ["smv", "tetragonal", "home-source"]}, + {"id": "3r6o", "input": "3r6o/series/219594b8_x0001.img", "wavelength": 1.54178, "ref": {"sg": "I 41", "sgno": 80, "cell": [90.68, 90.68, 76.13, 90.0, 90.0, 90.0], "dmin": 1.95}, "symmetry_evidence": "symmetry/3r6o", "reference_mismatch": "the frames are a crystal twinned ~30% (law k,h,-l, confirmed with XDS on the same frames); the deposited data are an untwinned crystal of the same form", "tags": ["smv", "tetragonal", "home-source"]}, {"id": "5cc8", "input": "5cc8/data/263060g10_x0001.img", "wavelength": 1.54178, "ref": {"sg": "P 21 21 2", "sgno": 18, "cell": [87.14, 93.76, 72.49, 90.0, 90.0, 90.0], "dmin": 1.75}, "symmetry_evidence": "symmetry/5cc8", "tags": ["smv", "orthorhombic", "home-source", "tncs"]}, {"id": "5ebi", "input": "5ebi/dna-rna-chimera_Ba_high_2_001.img", "wavelength": 0.82657, "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [35.72, 44.1, 35.72, 90.0, 119.98, 90.0], "dmin": 1.09}, "pinned": true, "symmetry_evidence": "symmetry/5ebi", "tags": ["marCCD", "monoclinic", "twin"]}, {"id": "5epe", "input": "5epe/030805_5epe/data/E1_7_set.001", "wavelength": 0.97856, "ref": {"sg": "F 2 3", "sgno": 196, "cell": [157.536, 157.536, 157.536, 90.0, 90.0, 90.0], "dmin": 1.9}, "tags": ["marCCD", "cubic"]}, diff --git a/tools/battery/report.py b/tools/battery/report.py index 06ea151bf..c9475c4bd 100644 --- a/tools/battery/report.py +++ b/tools/battery/report.py @@ -523,6 +523,9 @@ def build(run_dir, baseline=None, allow_incomplete=False): "REFMAC check (--model-check): R_free of the deposited model against our merge / against " "the deposited structure factors, both on the depositor's free set with the same protocol, " "one point per set, sorted; below 1 = our data fit the model better") + skipped = [r for r in res if r.get("refmac_status") == "skipped" and r["arm"] == "open"] + if skipped: + doc.p("REFMAC check skipped: " + "; ".join(f"{r['set']} ({r['refmac_reason']})" for r in skipped)) dep = [r for r in res if r.get("dep_status") == "ok"] if dep: @@ -544,9 +547,9 @@ def build(run_dir, baseline=None, allow_incomplete=False): doc.table(["set", "dep. data", "our d_min", "dep. d_min", "common refl.", "dCC all", "dCC outer 2", "CC past limit", "to d A", "REFMAC ratio", "ISa", "tags"], rows, num=range(2, 11)) - other = [r for r in res if r.get("dep_status") not in (None, "ok")] - if other: - doc.p("No comparison: " + "; ".join(f"{r['set']} ({r['dep_reason']})" for r in other)) + other = [r for r in res if r.get("dep_status") not in (None, "ok")] + if other: + doc.p("No depositor-data comparison: " + "; ".join(f"{r['set']} ({r['dep_reason']})" for r in other)) xds = [r for r in res if r.get("refres_range")] if xds: