diff --git a/tools/battery/README.md b/tools/battery/README.md index a16dfe411..a07e48c01 100644 --- a/tools/battery/README.md +++ b/tools/battery/README.md @@ -258,6 +258,19 @@ set (`refmac_reason` says why). The runner records `refmac_rfree`, `refmac_rwork `model_check.py --workdir DIR` runs it on one merge and prints every field (setting, reindexing operators of both data sets, what the depositor's amplitudes were made from). +### Rechecking an existing run + +`battery.py recheck RUN [--only a,b] [--jobs N]` reruns the model check and the depositor-data +comparison on every open-arm row of a finished run, from the run's own `p.mtz` (CPU only; rugnux is +not run again), so numbers made by an older `model_check.py` / `depdata_check.py` can be brought up +to date without reprocessing. It updates the `refmac_*` and `dep_*` fields of `results.json` in +place and nothing else (verdicts stay as scored). The previous files stay beside the new ones: +`results.json.pre-recheck-`, `report.{md,html}.pre-recheck-` and each set's +`model_check.pre-recheck-/`. The report is re-rendered, and `manifest.json` gains an entry +in its `rechecks` list - `date`, `end`, `runner_git`, `runner_dirty` (the tools version that made +the new numbers), `sets`, `fields`, `previous_results` and `command` - so which version of the +checks produced which figures stays auditable. The run directory is made read-only again. + ### The depositor-data comparison (open arm, always on) `tools/battery/depdata_check.py` compares our merge with the depositor's own data without CCP4: diff --git a/tools/battery/battery.py b/tools/battery/battery.py index a7c6de817..6cbfd547e 100644 --- a/tools/battery/battery.py +++ b/tools/battery/battery.py @@ -18,6 +18,8 @@ Usage (README.md in this directory is the full guide): battery.py compare RUN_A RUN_B [--all] [--rerun-changed] battery.py report RUN --out DIR [--baseline RUN] re-render a run's report into DIR/.{md,html} battery.py abort RUN [--reason TEXT] mark an unfinished run as aborted + battery.py recheck RUN [--only a,b] [--jobs N] rerun the model checks of a finished run's open + arm on its p.mtz files; recorded in the manifest battery.py discover [--refresh] open arm: propose inputs for new directories battery.py refs --arm inhouse|private [--write] re-read the XDS references from CORRECT.LP battery.py remap --arm A MAP.json [--write] follow a rename of the data directories @@ -37,6 +39,7 @@ A full-arm run (no --only / --tier) takes the site lock and refuses to start if A run that did not finish is never compared or reported on unless asked (--allow-incomplete). """ import argparse +import concurrent.futures import datetime import fcntl import hashlib @@ -519,6 +522,76 @@ def cmd_report(a, site): print(write_report(a.run, a.baseline, a.out, a.allow_incomplete)) +def recheck_one(e, wd, pdb_cache): + """The post-processing checks of one open-arm row, on the run's own p.mtz (a worker process).""" + out = {} + if e["model"] and os.path.exists(os.path.join(wd, "p.mtz")): + out.update(depdata_check.check_set(wd, e["id"], pdb_cache)) + out.update(check_model(e, wd)) + return out + + +def set_writable(paths): + for p in paths: + os.chmod(p, os.stat(p).st_mode | stat.S_IWUSR) + + +def cmd_recheck(a, site): + """Rerun model_check / depdata_check on a finished run's open-arm merges and update its + results.json in place. The previous results.json is kept beside it and the recheck is recorded + in manifest.json ("rechecks": which runner version produced the refmac_* / dep_* numbers).""" + run = os.path.abspath(a.run) + report.load_run(run, False) # refuses a run that did not finish + # the raw files: load_run rescores rows, and results.json keeps the verdicts as they were scored + man = json.load(open(os.path.join(run, "manifest.json"))) + results = json.load(open(os.path.join(run, "results.json"))) + only = set(a.only.split(",")) if a.only else None + rows = [r for r in results if r["arm"] == "open" and (only is None or r["set"] in only)] + if only and len(rows) != len(only): + sys.exit("not open-arm rows of this run: " + ", ".join(sorted(only - {r["set"] for r in rows}))) + if not rows: + sys.exit("nothing to recheck") + stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + entry = {"date": datetime.datetime.now().isoformat(timespec="seconds"), + "runner_git": git(HERE, "rev-parse", "--short", "HEAD") or "unknown", + "runner_dirty": bool(git(HERE, "status", "--porcelain", "--", ".")), + "sets": [r["set"] for r in rows], + "fields": list(REFMAC_KEYS + depdata_check.KEYS), + "previous_results": "results.json.pre-recheck-" + stamp, + "command": " ".join(sys.argv)} + pdb_cache = site.get("pdb_cache") or model_check.DEFAULT_CACHE + # a finished run directory is read-only: open up only what the recheck writes + wds = [os.path.join(run, "work", "open", r["set"]) for r in rows] + set_writable([run, os.path.join(run, "results.json"), os.path.join(run, "manifest.json")] + wds) + for wd in wds: # the previous check's files stay, renamed + if os.path.isdir(os.path.join(wd, "model_check")): + set_writable([os.path.join(wd, "model_check")]) + os.rename(os.path.join(wd, "model_check"), os.path.join(wd, "model_check.pre-recheck-" + stamp)) + shutil.copy2(os.path.join(run, "results.json"), os.path.join(run, "results.json.pre-recheck-" + stamp)) + with concurrent.futures.ProcessPoolExecutor(a.jobs) as pool: + futures = {pool.submit(recheck_one, {"id": r["set"], "model": r.get("model")}, wd, pdb_cache): r + for r, wd in zip(rows, wds)} + for n, fut in enumerate(concurrent.futures.as_completed(futures), 1): + r = futures[fut] + r.update(fut.result()) + print(f"[{n}/{len(rows)}] {r['set']:24s} {r['refmac_status'] or '-':8s} " + f"R_free depflags {report.f(r.get('refmac_rfree_depflags'), '{:.4f}')} " + f"depdata {report.f(r.get('refmac_rfree_depdata'), '{:.4f}')} " + f"{r.get('refmac_reason') or ''}", + flush=True) + write_json(os.path.join(run, "results.json"), results) + man.setdefault("rechecks", []).append(dict(entry, end=datetime.datetime.now().isoformat(timespec="seconds"))) + write_json(os.path.join(run, "manifest.json"), man) + for name in ("report", "report_PRIVATE"): + for ext in (".md", ".html"): + if os.path.exists(os.path.join(run, name + ext)): + set_writable([os.path.join(run, name + ext)]) + os.rename(os.path.join(run, name + ext), + os.path.join(run, f"{name}{ext}.pre-recheck-{stamp}")) + print(f"report {write_report(run, man.get('baseline'))}", flush=True) + make_read_only(run) + + def cmd_discover(a, site): """Propose an input for every open-arm directory the manifest does not have yet; with --refresh, also show where today's discovery disagrees with the manifest's (unpinned) input.""" @@ -655,6 +728,11 @@ def main(): ab.add_argument("--reason", default="aborted by hand") ab.add_argument("--force", action="store_true", help="even if its runner still seems alive") + rc = sub.add_parser("recheck") + rc.add_argument("run") + rc.add_argument("--only", help="comma-separated set ids (default: every open-arm row)") + rc.add_argument("--jobs", type=int, default=8, help="sets checked in parallel (CPU; REFMAC)") + d = sub.add_parser("discover") d.add_argument("--refresh", action="store_true") @@ -674,7 +752,7 @@ def main(): report.MANIFESTS = {arm: {e["id"]: e for e in load_sets(site, arm)} for arm, cfg in site["arms"].items() if os.path.exists(cfg["manifest"])} {"run": cmd_run, "list": cmd_list, "compare": cmd_compare, "report": cmd_report, - "abort": cmd_abort, "discover": cmd_discover, "refs": cmd_refs, "remap": cmd_remap}[a.cmd](a, site) + "abort": cmd_abort, "recheck": cmd_recheck, "discover": cmd_discover, "refs": cmd_refs, "remap": cmd_remap}[a.cmd](a, site) if __name__ == "__main__":