leonarski_fandClaude Opus 5.5
9b7a91d521
depdata_check.py compares each open-arm merge with the depositor's own data, per resolution shell, using the deposited model as the common yardstick: the rank correlation of our IMEAN with |Fc|^2 minus that of the deposited intensities (or F^2), on the reflections both carry (d < 4 A, eight equal-count shells), with the data carried into the model's setting by model_check's change-of-basis search. New row keys dep_cc_delta_all, dep_cc_delta_outer (two outermost common shells), dep_beyond_cc / dep_beyond_d (our correlation with |Fc|^2 past the deposited data's limit), dep_kind, dep_d_min, dep_n_common, dep_status, dep_reason. gemmi only, no CCP4; runs on every open-arm set with a model, and `report` fills it in for older runs (four processes). Reported in its own report section and the per-set table, never scored. Rank rather than Pearson correlation: on the phase-1 run a handful of gross outliers (I/sigma above 1000 at 2.4 A, or ~1000x the shell median at 1.41 A) decided the outer-shell Pearson CC of several merges on their own. On the phase-1 run (38 of 40 open-arm sets compared; the two without are the lattice failures) the outer-shell delta has median -0.012; 22 s for the whole report fill-in. model_check.py: REFMAC's mmCIF reader stopped with "rdaniso_cif: Atom symbol mismatch" on 12 of 35 entries whose atoms carry ANISOU only in part. The model is now written in the PDB format where it fits (under 100000 atoms, chain names of at most two characters), where each ANISOU follows its own ATOM line; all 12 then score. R-factors of the entries that worked before move by < 0.001. The runner also records model_check's baseline note (e.g. an intensity-only deposition) in refmac_reason instead of leaving it empty. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
593 lines
33 KiB
Python
593 lines
33 KiB
Python
"""The per-run report and the run-to-run comparison."""
|
|
import json
|
|
import multiprocessing
|
|
import os
|
|
import socket
|
|
import statistics
|
|
import sys
|
|
|
|
import depdata_check
|
|
import score
|
|
from render import Doc, ratio_dots, verdict_bars
|
|
|
|
# A change smaller than these is treated as run-to-run noise. They are a starting point, to be
|
|
# re-calibrated from `compare --rerun-changed`, which measures the noise directly.
|
|
NOISE = {
|
|
"d_min": ("rel", 0.02), # 2% of the resolution limit
|
|
"isa": ("rel", 0.05),
|
|
"r_meas": ("rel", 0.05),
|
|
"cc_half": ("abs", 0.005),
|
|
"completeness": ("abs", 1.0), # percentage points
|
|
"cell_dev_pct": ("abs", 0.2), # percentage points
|
|
"rfree": ("abs", 0.005),
|
|
"rmodel_shell_scaled": ("abs", 0.005),
|
|
"radial_misfit": ("abs", 0.05),
|
|
"refres_r_meas": ("rel", 0.05),
|
|
"refres_isa": ("rel", 0.05),
|
|
"refres_lowres_r_meas": ("rel", 0.05),
|
|
"wall_s": ("rel", 0.25), # and at least 10 s - see beyond_noise()
|
|
}
|
|
|
|
|
|
def run_state(man):
|
|
"""complete, aborted (marked so), running (its runner is alive), or unfinished (killed
|
|
without a trace - treat as aborted)."""
|
|
if man.get("complete"):
|
|
return "complete"
|
|
if man.get("aborted"):
|
|
return "aborted"
|
|
if man.get("pid") and man.get("host") == socket.gethostname():
|
|
try:
|
|
os.kill(man["pid"], 0)
|
|
return "running"
|
|
except OSError:
|
|
pass
|
|
return "unfinished"
|
|
|
|
|
|
def load_run(run_dir, allow_incomplete=False):
|
|
"""A run's manifest and results. A run that did not finish covers an arbitrary part of its
|
|
sets and is refused unless asked for."""
|
|
man = json.load(open(os.path.join(run_dir, "manifest.json")))
|
|
state = run_state(man)
|
|
if state != "complete" and not allow_incomplete:
|
|
sys.exit(f"{run_dir} is {state}, not a finished run; pass --allow-incomplete to use it anyway")
|
|
path = os.path.join(run_dir, "results.json")
|
|
res = json.load(open(path)) if os.path.exists(path) else []
|
|
schema = man.get("results_schema", 1)
|
|
if schema < 2:
|
|
schema1(man, res)
|
|
if schema < 3:
|
|
res[:] = schema2(res)
|
|
else:
|
|
rescore(run_dir, man, res)
|
|
return man, res
|
|
|
|
|
|
def rescore(run_dir, man, res):
|
|
"""Score every row again from its saved report, with today's scorer and today's manifest rows
|
|
(references, `unscored`, `expect`), so a report or a compare never mixes two scorings or two
|
|
references. results.json keeps the verdicts as they were scored when the run finished.
|
|
|
|
A row whose report has no lattice keeps its verdict: why it failed (crash, reader, timeout)
|
|
came from the run, not from the report."""
|
|
own = {(e["arm"], e["id"]): e for e in man["sets"]}
|
|
for r in res:
|
|
e = MANIFESTS.get(r["arm"], {}).get(key(r)[1]) or own.get((r["arm"], r["set"]))
|
|
rep = score.read_report(os.path.join(run_dir, "work", r["arm"], r["set"], "p_report.txt"))
|
|
lattice = rep.get("SPACE_GROUP_NUMBER") and rep.get("UNIT_CELL_CONSTANTS")
|
|
if e and (lattice or e.get("expect") == "no_lattice"):
|
|
r.update(score.judge(dict(e, arm=r["arm"]), rep, ""))
|
|
|
|
|
|
def _depdata(job):
|
|
os.nice(10)
|
|
return depdata_check.check_set(*job)
|
|
|
|
|
|
def fill_depdata(run_dir, res):
|
|
"""The depositor-data comparison (depdata_check.py) for open-arm rows of a run made before it
|
|
existed, computed from the saved merge, four sets at a time."""
|
|
todo = [r for r in res if r["arm"] == "open" and "dep_status" not in r and r.get("model")
|
|
and os.path.exists(os.path.join(run_dir, "work", "open", r["set"], "p.mtz"))]
|
|
if not todo:
|
|
return
|
|
print(f"depositor-data comparison for {len(todo)} sets of an older run ...", file=sys.stderr)
|
|
jobs = [(os.path.join(run_dir, "work", "open", r["set"]), r["set"],
|
|
PDB_CACHE or depdata_check.model_check.DEFAULT_CACHE) for r in todo]
|
|
with multiprocessing.Pool(4) as pool:
|
|
for r, out in zip(todo, pool.map(_depdata, jobs, chunksize=1)):
|
|
r.update(out)
|
|
|
|
|
|
def schema1(man, res):
|
|
"""Read a run from before the variants (results schema 1) as one: it ran each set once, plain
|
|
on the open arm and with XDS's settings on the XDS arms - unless --unforced, which left the
|
|
range free but still passed -A, and is read as `bare`. Its R-factors came from the REFMAC check."""
|
|
for r in res:
|
|
r.setdefault("variant", "xds" if r["arm"] != "open" and man["options"].get("xds_range") else "bare")
|
|
if "model_status" in r:
|
|
for k in ("rfree", "rwork", "rfree_depflags", "rfree_depdata", "rfree_ratio"):
|
|
r["refmac_" + k] = r.pop(k, None)
|
|
r["refmac_status"], r["refmac_reason"] = r.pop("model_status"), r.pop("model_reason", None)
|
|
r["rfree_deposited"] = None
|
|
|
|
|
|
def schema2(res):
|
|
"""Read a run with variants (schema 2) as one run per set: its `bare` row, with the open arm's
|
|
`model` row's R-factors folded in. That is today's processing on the open arm; on the XDS arms
|
|
today's command adds -A where XDS was anomalous. The `xds` rows (XDS's range forced on the
|
|
processing) have no counterpart and are left out."""
|
|
model = {r["set"]: r for r in res if r["variant"] == "model"}
|
|
out = []
|
|
for r in res:
|
|
if r["variant"] != "bare":
|
|
continue
|
|
m = model.get(r["set"])
|
|
if m:
|
|
r.update({k: m.get(k) for k in ("rfree", "rwork", "model_fit", "rfree_deposited", "rfree_ratio")})
|
|
out.append(r)
|
|
return out
|
|
|
|
|
|
def expected_rows(man):
|
|
"""Rows a finished run has: one per set (per set and variant before schema 3)."""
|
|
if man.get("results_schema", 1) >= 3:
|
|
return len(man["sets"])
|
|
return sum(len(man.get("variants", {}).get(e["arm"], [None])) for e in man["sets"])
|
|
|
|
|
|
def check_baseline(baseline, private):
|
|
"""A baseline must have finished, and be as private as the run it is compared with: the delta
|
|
table names its sets."""
|
|
man, _ = load_run(baseline)
|
|
if bool(man.get("private")) != bool(private):
|
|
sys.exit(f"baseline {baseline} is {'' if man.get('private') else 'not '}a private run; "
|
|
"a report may only be compared with a run of its own kind")
|
|
|
|
|
|
# Version of the results.json row schema (README.md, "results.json"). Bump it when a field changes
|
|
# meaning or is removed; adding a field does not need it.
|
|
# 2: one row per set AND variant (`variant`, `first_read`); `rfree`/`rwork`/`rfree_ratio` are rugnux's
|
|
# own --model R-factors, the REFMAC check's moved to `refmac_*`.
|
|
# 3: one row per set again (one command per arm, no `variant`/`first_read`); `refres_*` added;
|
|
# `sgno`/`sg` are the data's own group where --model relabelled the hand (`sg_label`).
|
|
RESULTS_SCHEMA = 3
|
|
|
|
# {arm: {old set id: current id}} from the manifests (battery.py sets it), so a run made before a
|
|
# dataset was renamed still pairs with one made after
|
|
ALIASES = {}
|
|
|
|
# {arm: {set id: manifest row}} of today's manifests (battery.py sets it); rescore() scores with them
|
|
MANIFESTS = {}
|
|
|
|
# the site's cache of deposited models and structure factors (battery.py sets it)
|
|
PDB_CACHE = None
|
|
|
|
|
|
def key(r):
|
|
return (r["arm"], ALIASES.get(r["arm"], {}).get(r["set"], r["set"]))
|
|
|
|
|
|
def f(x, fmt="{:.2f}", dash="-"):
|
|
return dash if x is None else fmt.format(x)
|
|
|
|
|
|
def pct(x, nd=1):
|
|
return "-" if x is None else f"{100 * x:.{nd}f}%"
|
|
|
|
|
|
def median(xs):
|
|
xs = [x for x in xs if x is not None]
|
|
return statistics.median(xs) if xs else None
|
|
|
|
|
|
def quartiles(xs):
|
|
xs = sorted(x for x in xs if x is not None)
|
|
if not xs:
|
|
return None
|
|
if len(xs) < 4:
|
|
return xs[0], xs[len(xs) // 2], xs[len(xs) // 2], xs[len(xs) // 2], xs[-1]
|
|
q = statistics.quantiles(xs, n=4)
|
|
return xs[0], q[0], q[1], q[2], xs[-1]
|
|
|
|
|
|
def beyond_noise(name, a, b):
|
|
if a is None or b is None:
|
|
return (a is None) != (b is None)
|
|
kind, tol = NOISE[name]
|
|
d = abs(b - a)
|
|
if name == "wall_s" and d < 10:
|
|
return False
|
|
return d > (tol * abs(a) if kind == "rel" else tol)
|
|
|
|
|
|
def changes(ra, rb):
|
|
"""What changed between two results of the same set, beyond noise."""
|
|
out = []
|
|
if ra["verdict"] != rb["verdict"]:
|
|
out.append(f"verdict {ra['verdict']}->{rb['verdict']}")
|
|
if ra.get("sgno") != rb.get("sgno"):
|
|
out.append("space group")
|
|
if ra.get("volume_ratio") and rb.get("volume_ratio") and \
|
|
abs(ra["volume_ratio"] - rb["volume_ratio"]) > 0.05:
|
|
out.append("lattice")
|
|
for name in ("d_min", "isa", "r_meas", "cc_half", "completeness", "cell_dev_pct"):
|
|
if beyond_noise(name, ra.get(name), rb.get(name)):
|
|
out.append(name)
|
|
# The R that is comparable between runs, and the radial profile that decides whether `rfree`
|
|
# beside it can be read at all (both only where both runs carry them).
|
|
for name in ("rmodel_shell_scaled", "radial_misfit", "rfree"):
|
|
if ra.get(name) is not None and rb.get(name) is not None and beyond_noise(name, ra[name], rb[name]):
|
|
out.append(name)
|
|
# the like-for-like table only where both runs have one (a pre-schema-3 run has none)
|
|
for name in ("refres_r_meas", "refres_isa", "refres_lowres_r_meas"):
|
|
if ra.get(name) is not None and rb.get(name) is not None and beyond_noise(name, ra[name], rb[name]):
|
|
out.append(name)
|
|
if beyond_noise("wall_s", ra.get("wall_s"), rb.get("wall_s")):
|
|
out.append("time")
|
|
return out
|
|
|
|
|
|
def compare_rows(res_a, res_b):
|
|
"""[(key, ra, rb, changes)] over the sets both runs have, then those only one has."""
|
|
a = {key(r): r for r in res_a}
|
|
b = {key(r): r for r in res_b}
|
|
rows = [(k, a[k], b[k], changes(a[k], b[k])) for k in sorted(a) if k in b]
|
|
rows += [(k, a[k], None, ["only in A"]) for k in sorted(a) if k not in b]
|
|
rows += [(k, None, b[k], ["only in B"]) for k in sorted(b) if k not in a]
|
|
return rows
|
|
|
|
|
|
def arrow(x, y, fmt=lambda v: str(v)):
|
|
x = "-" if x is None else fmt(x)
|
|
y = "-" if y is None else fmt(y)
|
|
return x if x == y else f"{x} -> {y}"
|
|
|
|
|
|
def compare_table(doc, rows, only_changed=True):
|
|
head = ["set", "arm", "verdict", "space group", "d_min", "ISa", "R_meas", "CC1/2",
|
|
"cell dev %", "R_model (shell-scaled)", "R_free", "radial misfit",
|
|
"ref-range R_meas", "ref-range low-res R_meas", "time s", "beyond noise"]
|
|
out = []
|
|
for (arm, s), ra, rb, ch in rows:
|
|
if only_changed and not ch:
|
|
continue
|
|
ra, rb = ra or {}, rb or {}
|
|
out.append([s, arm, arrow(ra.get("verdict"), rb.get("verdict")),
|
|
arrow(ra.get("sg"), rb.get("sg")),
|
|
arrow(ra.get("d_min"), rb.get("d_min"), lambda v: f"{v:.2f}"),
|
|
arrow(ra.get("isa"), rb.get("isa"), lambda v: f"{v:.1f}"),
|
|
arrow(ra.get("r_meas"), rb.get("r_meas"), lambda v: f"{100 * v:.1f}%"),
|
|
arrow(ra.get("cc_half"), rb.get("cc_half"), lambda v: f"{v:.3f}"),
|
|
arrow(ra.get("cell_dev_pct"), rb.get("cell_dev_pct"), lambda v: f"{v:.2f}"),
|
|
arrow(ra.get("rmodel_shell_scaled"), rb.get("rmodel_shell_scaled"), lambda v: f"{v:.3f}"),
|
|
arrow(ra.get("rfree"), rb.get("rfree"), lambda v: f"{v:.3f}"),
|
|
arrow(ra.get("radial_misfit"), rb.get("radial_misfit"), lambda v: f"{v:.2f}"),
|
|
arrow(ra.get("refres_r_meas"), rb.get("refres_r_meas"), lambda v: f"{100 * v:.1f}%"),
|
|
arrow(ra.get("refres_lowres_r_meas"), rb.get("refres_lowres_r_meas"),
|
|
lambda v: f"{100 * v:.1f}%"),
|
|
arrow(ra.get("wall_s"), rb.get("wall_s"), lambda v: f"{v:.0f}"),
|
|
", ".join(ch)])
|
|
doc.table(head, out, row_class=lambda r: "fail" if "verdict" in r[-1] else "")
|
|
|
|
|
|
def crystals(rs):
|
|
"""Rows that describe a crystal - the no-crystal controls would only pollute a distribution."""
|
|
return [r for r in rs if "control" not in r.get("tags", [])]
|
|
|
|
|
|
def scored(rs):
|
|
return [r for r in rs if r["verdict"] in ("pass", "fail")]
|
|
|
|
|
|
def status(r):
|
|
"""The verdict, except that a pass carried by an accepted alternative reference (score.py,
|
|
`alternatives`) is counted on its own, so a reader sees how many rows the battery passed on a
|
|
second acceptable answer rather than on the deposition."""
|
|
return "accepted_alt" if r.get("cause") == "accepted_alternative" else r["verdict"]
|
|
|
|
|
|
def rate(rs):
|
|
s = scored(rs)
|
|
n = sum(r["verdict"] == "pass" for r in s)
|
|
return f"{n}/{len(s)} ({100 * n / len(s):.0f}%)" if s else "-"
|
|
|
|
|
|
def build(run_dir, baseline=None, allow_incomplete=False):
|
|
"""Markdown and HTML report of one run, with the delta against a baseline run if given."""
|
|
man, res = load_run(run_dir, allow_incomplete)
|
|
if baseline:
|
|
check_baseline(baseline, man.get("private"))
|
|
fill_depdata(run_dir, res)
|
|
b = man["binary"]
|
|
doc = Doc(f"Rugnux battery - {os.path.basename(os.path.normpath(run_dir))}")
|
|
if man.get("private"):
|
|
doc.p("PRIVATE ARM - confidential user data. This report stays in the run directory; it "
|
|
"must not be copied into the repository, an issue or any public report.")
|
|
state = run_state(man)
|
|
if state != "complete":
|
|
why = f" ({man['aborted']['reason']})" if man.get("aborted") else ""
|
|
doc.p(f"RUN {state.upper()}{why}: {len(res)} of {expected_rows(man)} rows have a "
|
|
"result. Not a reference and not comparable with a finished run.")
|
|
if man.get("results_schema", 1) < 3:
|
|
doc.p("A run from before the one-command battery: shown by its `bare` rows (with the `model` "
|
|
"rows' R-factors on the open arm); its `xds` rows have no counterpart and are left out.")
|
|
doc.ul([
|
|
f"rugnux {b.get('version', '?')}, sha256 {b['sha256'][:16]}, build flags: {b.get('flags') or 'unknown'}",
|
|
f"runner {man['runner_git']}, host {man['host']}, {man['start']} -> {man.get('end') or 'NOT FINISHED'}",
|
|
f"arms: {', '.join(man['arms'])}; {len(man['sets'])} sets"
|
|
+ (f" (subset: {man['subset']})" if man.get("subset") else " (full arm)"),
|
|
"timing: NOT a reference - the GPU was shared" if man.get("gpu_shared")
|
|
else "timing: GPU was not shared during the run",
|
|
f"command options: {' '.join(man['options']['extra_args']) or '(none beyond output selection)'}",
|
|
] + [f"command, {arm} arm: {d}" for arm, d in man.get("command_doc", {}).items()])
|
|
|
|
arms = [arm for arm in man["arms"] if any(r["arm"] == arm for r in res)]
|
|
order = {arm: i for i, arm in enumerate(arms)}
|
|
|
|
doc.h(2, "Summary")
|
|
head = ["arm", "sets", "pass", "of which alt", "fail", "unscored", "not run", "pass rate",
|
|
"median res. gain", "median ISa", "median R_meas", "median time s", "total time min"]
|
|
rows = []
|
|
for arm in arms:
|
|
rs = [r for r in res if r["arm"] == arm]
|
|
cnt = {x: sum(r["verdict"] == x for r in rs) for x in ("pass", "fail", "unscored", "not_run")}
|
|
cnt["accepted_alt"] = sum(status(r) == "accepted_alt" for r in rs)
|
|
xs = crystals(rs)
|
|
rows.append([arm, len(rs), cnt["pass"], cnt["accepted_alt"], cnt["fail"], cnt["unscored"],
|
|
cnt["not_run"], rate(rs),
|
|
f(median(r.get("res_gain_pct") for r in xs), "{:+.1f}%"),
|
|
f(median(r.get("isa") for r in xs), "{:.1f}"),
|
|
pct(median(r.get("r_meas") for r in xs)),
|
|
f(median(r.get("wall_s") for r in rs), "{:.0f}"),
|
|
f(sum(r.get("wall_s") or 0 for r in rs) / 60, "{:.1f}")])
|
|
doc.table(head, rows, num=range(1, 13))
|
|
doc.svg(verdict_bars([(arm, {x: sum(status(r) == x and r["arm"] == arm for r in res)
|
|
for x in ("pass", "accepted_alt", "fail", "unscored", "not_run")})
|
|
for arm in arms]), "verdicts per arm")
|
|
doc.p("Pass rate is over the scored sets (pass + fail). `of which alt` counts the passes that "
|
|
"are already in `pass`: rows where the answer matched an accepted alternative reference "
|
|
"rather than the deposition, listed under Accepted alternatives below. In the bar chart "
|
|
"they are drawn as their own segment. Every number in this table is rugnux's own "
|
|
"run at its own resolution cut: what a user gets. Resolution gain is (reference d_min - our "
|
|
"d_min) / reference d_min: positive = finer than the deposition, or than XDS. Medians leave "
|
|
"out the no-crystal controls. R_meas and CC1/2 are pooled over each run's own resolution "
|
|
"range, so they describe a run and do not rank two; the like-for-like comparison with XDS "
|
|
"is the reference-range table below. Each set runs once, so every time includes reading "
|
|
"its images from disk (unless they were still in the page cache from before the run).")
|
|
|
|
doc.h(3, "By population")
|
|
rows = []
|
|
for arm in arms:
|
|
rs = [r for r in res if r["arm"] == arm]
|
|
for tag in sorted({t for r in rs for t in r.get("tags", [])}):
|
|
sub = [r for r in rs if tag in r.get("tags", [])]
|
|
rows.append([arm, tag, len(sub), rate(sub), f(median(r.get("res_gain_pct") for r in sub), "{:+.1f}%")])
|
|
doc.table(["arm", "population", "sets", "pass rate", "median res. gain"], rows, num=(2, 4))
|
|
|
|
doc.h(3, "Distributions (sets that merged, controls left out)")
|
|
rows = []
|
|
for arm in arms:
|
|
rs = crystals(r for r in res if r["arm"] == arm)
|
|
for name, lab, fm in (("isa", "ISa", "{:.1f}"), ("r_meas", "R_meas", None),
|
|
("cc_half", "CC1/2", "{:.3f}"), ("res_gain_pct", "res. gain %", "{:+.1f}"),
|
|
("isa_ratio", "ref-range ISa / XDS", "{:.3f}"),
|
|
("r_meas_ratio", "ref-range R_meas / XDS", "{:.3f}"),
|
|
("lowres_r_meas", "low-res shell R_meas", None),
|
|
("lowres_r_meas_ratio", "ref-range low-res R_meas / XDS", "{:.3f}"),
|
|
("cc_half_noise_ratio", "ref-range CC1/2 noise / XDS", "{:.2f}"),
|
|
("rmodel_shell_scaled", "R_model, shell-scaled", "{:.3f}"),
|
|
("rfree", "R_free (placement, within-run only)", "{:.3f}"),
|
|
("radial_misfit", "radial misfit", "{:.2f}"),
|
|
("rfree_ratio", "R_free ratio", "{:.3f}"),
|
|
("refmac_rfree_ratio", "REFMAC R_free ratio", "{:.3f}"),
|
|
("dep_cc_delta_all", "dCC vs depositor, all", "{:+.3f}"),
|
|
("dep_cc_delta_outer", "dCC vs depositor, outer 2 shells", "{:+.3f}"),
|
|
("dep_beyond_cc", "CC(Fc^2) past the deposited limit", "{:.3f}"),
|
|
("wall_s", "time s", "{:.0f}")):
|
|
q = quartiles(r.get(name) for r in rs)
|
|
if q:
|
|
cells = [pct(x) if fm is None else f(x, fm) for x in q]
|
|
rows.append([arm, lab, sum(r.get(name) is not None for r in rs)] + cells)
|
|
doc.table(["arm", "metric", "n", "min", "q1", "median", "q3", "max"], rows, num=range(2, 8))
|
|
|
|
doc.h(3, "Per set, against the reference")
|
|
for arm in arms:
|
|
pts = [(r["set"], r["d_min"] / r["d_min_ref"], r["verdict"],
|
|
f"d_min {r['d_min']:.2f} vs {r['d_min_ref']:.2f} A, {r['verdict']}")
|
|
for r in res if r["arm"] == arm and r.get("d_min") and r.get("d_min_ref")]
|
|
if pts:
|
|
doc.svg(ratio_dots(pts, f"{arm}: d_min ratio"),
|
|
f"{arm}: d_min(rugnux, own cut) / d_min(reference), one point per set, sorted; below 1 "
|
|
"= finer than the reference")
|
|
for name, what, cap in (
|
|
("isa_ratio", "ISa", "ISa of the reference-range table (the error model refitted on it) / "
|
|
"XDS's ISa; above 1 = rugnux's error model is the better one"),
|
|
("r_meas_ratio", "R_meas", "R_meas over the reference range / XDS's R_meas; below 1 = "
|
|
"rugnux's merge is the more consistent one"),
|
|
("lowres_r_meas_ratio", "low-res R_meas", "R_meas of the lowest-resolution shell of the "
|
|
"reference-range table / of XDS's table; below 1 = "
|
|
"rugnux's strong reflections agree better"),
|
|
("cc_half_noise_ratio", "CC1/2 noise", "half-set noise over the reference range read off CC1/2 "
|
|
"(1 / CC1/2 - 1) / XDS's; 2 = as noisy as XDS's merge "
|
|
"would be with half its observations (not scored)")):
|
|
for arm in arms:
|
|
pts = [(r["set"], r[name], r["verdict"], f"{what} ratio, {refres_note(r)}")
|
|
for r in res if r["arm"] == arm and r.get(name)]
|
|
if pts:
|
|
doc.svg(ratio_dots(pts, f"{arm}: reference-range {what} ratio"),
|
|
f"{arm}: {cap}. One point per set, sorted.")
|
|
pts = [(r["set"], r["rfree_ratio"], r["verdict"],
|
|
f"R_free {r['rfree']:.3f} vs deposited {r['rfree_deposited']:.3f}")
|
|
for r in res if r.get("rfree_ratio")]
|
|
if pts:
|
|
doc.svg(ratio_dots(pts, "R_free ratio"),
|
|
"R_free of the deposited model against our merge, as rugnux reports it with --model (a "
|
|
"rigid-body placement, scored on rugnux's own free set: a trend number, not a refinement) / "
|
|
"the R_free the depositor published, one point per set, sorted; below 1 = lower than published")
|
|
pts = [(r["set"], r["refmac_rfree_ratio"], r["verdict"],
|
|
f"R_free {r['refmac_rfree_depflags']:.3f} vs {r['refmac_rfree_depdata']:.3f} on the deposited data")
|
|
for r in res if r.get("refmac_rfree_ratio")]
|
|
if pts:
|
|
doc.svg(ratio_dots(pts, "REFMAC R_free ratio"),
|
|
"REFMAC check (--model-check): R_free of the deposited model against our merge / against "
|
|
"the deposited structure factors, both on the depositor's free set with the same protocol, "
|
|
"one point per set, sorted; below 1 = our data fit the model better")
|
|
|
|
dep = [r for r in res if r.get("dep_status") == "ok"]
|
|
if dep:
|
|
doc.h(3, "Against the depositor's data")
|
|
doc.p("Per resolution shell, the rank correlation of our merged intensities with |Fc|^2 minus that of the depositor's own "
|
|
"data, on the reflections both carry (d < 4 A, eight shells of equal count), |Fc| from "
|
|
"the deposited model as it is (no bulk solvent, no refinement; depdata_check.py). The "
|
|
"model was refined against the depositor's data, so zero is already a good result: "
|
|
"across a corpus the median sits near -0.01. 'Past the limit' is CC(I, |Fc|^2) of our "
|
|
"reflections in the outermost of up to three shells beyond the deposited data's limit, "
|
|
"which the model never saw: clearly above zero there is signal. Reported, never scored.")
|
|
rows = []
|
|
for r in sorted(dep, key=lambda r: r["dep_cc_delta_outer"]):
|
|
rows.append([r["set"], r["dep_kind"], f(r.get("d_min")), f(r.get("dep_d_min")),
|
|
r["dep_n_common"], f(r["dep_cc_delta_all"], "{:+.3f}"),
|
|
f(r["dep_cc_delta_outer"], "{:+.3f}"), f(r.get("dep_beyond_cc"), "{:.3f}"),
|
|
f(r.get("dep_beyond_d")), f(r.get("refmac_rfree_ratio"), "{:.3f}"),
|
|
f(r.get("isa"), "{:.1f}"), ", ".join(r.get("tags", []))])
|
|
doc.table(["set", "dep. data", "our d_min", "dep. d_min", "common refl.", "dCC all",
|
|
"dCC outer 2", "CC past limit", "to d A", "REFMAC ratio", "ISa", "tags"],
|
|
rows, num=range(2, 11))
|
|
other = [r for r in res if r.get("dep_status") not in (None, "ok")]
|
|
if other:
|
|
doc.p("No comparison: " + "; ".join(f"{r['set']} ({r['dep_reason']})" for r in other))
|
|
|
|
xds = [r for r in res if r.get("refres_range")]
|
|
if xds:
|
|
doc.h(3, "Like for like with XDS: the reference-range table")
|
|
doc.p("The same merge, binned a second time over XDS's range (--report-resolution; report-only, "
|
|
"nothing was processed differently for it), against XDS's CORRECT.LP totals. Where "
|
|
"rugnux's own cut is coarser than the reference ('coverage' in the last column), the "
|
|
"shells past it are not merged at all: the completeness there is coverage of XDS's "
|
|
"range, not a quality loss, and the other rugnux numbers are over the shells it reached. "
|
|
"XDS's totals are over its own merged range, which is the reference range except where "
|
|
"the reference d_min was derived (see below).")
|
|
rows = []
|
|
for r in sorted(xds, key=lambda r: (order.get(r["arm"], 9), r["set"])):
|
|
rows.append([r["set"], r["arm"], r["refres_range"], f(r.get("d_min")),
|
|
f(r.get("refres_completeness"), "{:.1f}") + " / " + f(r.get("completeness_ref"), "{:.1f}"),
|
|
f(r.get("refres_multiplicity"), "{:.1f}") + " / " + f(r.get("multiplicity_ref"), "{:.1f}"),
|
|
f(r.get("refres_i_over_sigma"), "{:.1f}"),
|
|
pct(r.get("refres_r_meas")) + " / " + pct(r.get("r_meas_ref")),
|
|
lowres_cell(r),
|
|
f(r.get("refres_cc_half"), "{:.4f}") + " / " + f(r.get("cc_half_ref"), "{:.3f}"),
|
|
f(r.get("cc_half_noise_ratio"), "{:.2f}"),
|
|
f(r.get("refres_isa"), "{:.1f}") + " / " + f(r.get("isa_ref"), "{:.1f}"),
|
|
refres_note(r)])
|
|
doc.table(["set", "arm", "range A", "own d_min", "compl % (rugnux / XDS)", "mult", "I/sigma",
|
|
"R_meas", "low-res R_meas (to d A)", "CC1/2", "CC1/2 noise ratio", "ISa", "reading"],
|
|
rows, num=(3, 6, 10))
|
|
doc.p("Data quality is not scored; this table is for a human to judge. "
|
|
"CC1/2 = S / (S + E) (signal and half-set error variances), so 1 / CC1/2 - 1 = E / S is the "
|
|
"half-set noise; the noise ratio is rugnux's over XDS's, with XDS's CC1/2 taken at the "
|
|
"bottom of its printed rounding (-0.0005); 2 would mean noisier than XDS's merge with half "
|
|
"its observations. The low-resolution R_meas is the lowest shell of each table (the shells "
|
|
"are XDS's, so both cover the same reflections); reported like everything here.")
|
|
|
|
derived = {r["set"]: r for r in res if (r.get("d_min_ref_rule") or "xds_range") != "xds_range"}
|
|
if derived:
|
|
doc.p("XDS merged past its own signal (CC1/2 of its finest shell not significant), so the "
|
|
"reference d_min is where XDS's CC1/2 falls through 0.30, and that is also the d_min of the "
|
|
"reference-range table: "
|
|
+ "; ".join(f"{s} (XDS range {f(r.get('d_min_xds'))} A, reference "
|
|
f"{f(r.get('d_min_ref'))} A)" for s, r in sorted(derived.items()))
|
|
+ ". XDS's pooled R_meas, CC1/2 and completeness are over its whole range.")
|
|
|
|
doc.h(3, "Failures")
|
|
fails = [r for r in res if r["verdict"] == "fail"]
|
|
doc.table(["set", "arm", "cause", "reason"],
|
|
[[r["set"], r["arm"], r["cause"], r["reason"]] for r in fails])
|
|
other = [r for r in res if r["verdict"] in ("unscored", "not_run")]
|
|
if other:
|
|
doc.p("Not scored: " + "; ".join(f"{r['set']} ({r['reason']})" for r in other))
|
|
|
|
alt = [r for r in res if status(r) == "accepted_alt"]
|
|
if alt:
|
|
doc.h(3, "Accepted alternatives")
|
|
doc.p(f"{len(alt)} of the passes above are rows that accept more than one reference: the "
|
|
"deposition and our reduction disagree, the disagreement is real, and no test "
|
|
"available to us settles it, so either answer passes as long as the program picks "
|
|
"one of them. These are open questions, not errors attributed to the deposition; "
|
|
"the manifest's `ref` keeps the deposited values verbatim in every one of them.")
|
|
doc.ul(f"{r['set']} ({r['arm']}): we report {r.get('sg') or '-'}, the reference is "
|
|
f"{r.get('sg_ref') or '-'}; accepted as {r['accepted_alt']}. {r['accepted_alt_why']}"
|
|
for r in sorted(alt, key=lambda r: (r["arm"], r["set"])))
|
|
|
|
doc.h(3, "Per set")
|
|
doc.p("Space group: on the open arm the data's own determination. Where --model put the model's "
|
|
"enantiomorph on the label, the label is in the note and the group scored is the search's.")
|
|
if any(r.get("rmodel_shell_scaled") for r in res):
|
|
doc.p("R_free and R_work describe one run against its model and do not carry across runs: the "
|
|
"model is scaled to the data by an overall factor and an anisotropic B, so whatever the "
|
|
"amplitudes' radial profile does that this shape cannot follow is reported as R, and a "
|
|
"change in the profile alone moves R_free further than a real change in the data does. "
|
|
"R_model (shell-scaled) is the same R with one free scale per resolution shell removing "
|
|
"exactly that, over every reflection rather than the free 5%, and it is the column to "
|
|
"read in a delta table. Radial misfit is how big that per-shell rescale had to be (the "
|
|
"RMS of its logarithm): when it moves between two runs, R_free between them means "
|
|
"little. Both are reported for a human to judge; nothing is scored on them.")
|
|
model = [("R_model (shell-scaled)", "rmodel_shell_scaled"), ("radial misfit", "radial_misfit"),
|
|
("CC model", "cc_model"), ("R_free (within-run)", "rfree"), ("R_work", "rwork"),
|
|
("dep R_free", "rfree_deposited"),
|
|
("R_free ratio", "rfree_ratio")] if any(r.get("rfree") for r in res) else []
|
|
model += [("REFMAC R_free", "refmac_rfree_depflags"), ("REFMAC dep data", "refmac_rfree_depdata"),
|
|
("REFMAC ratio", "refmac_rfree_ratio")] if any(r.get("refmac_status") for r in res) else []
|
|
model += [("dCC dep. outer", "dep_cc_delta_outer")] if any(r.get("dep_status") for r in res) else []
|
|
head = ["set", "arm", "verdict", "space group", "ref", "cell dev %", "V ratio", "d_min", "ref d_min",
|
|
"gain", "compl %", "mult", "R_meas", "low-res R_meas", "CC1/2", "ISa", "ref ISa", "idx"] + \
|
|
[h for h, _ in model] + (["model fit"] if model else []) + ["time s", "note"]
|
|
rows = []
|
|
for r in sorted(res, key=lambda r: (order.get(r["arm"], 9), r["set"])):
|
|
note = "; ".join(x for x in (r.get("reason"), r.get("sg_label") and f"labelled {r['sg_label']} from the model")
|
|
if x)
|
|
rows.append([r["set"], r["arm"], r["verdict"], r.get("sg") or "-", r.get("sg_ref") or "-",
|
|
f(r.get("cell_dev_pct")), f(r.get("volume_ratio"), "{:.3f}"),
|
|
f(r.get("d_min")), f(r.get("d_min_ref")), f(r.get("res_gain_pct"), "{:+.1f}%"),
|
|
f(r.get("completeness"), "{:.1f}"), f(r.get("multiplicity"), "{:.1f}"),
|
|
pct(r.get("r_meas")), pct(r.get("lowres_r_meas")), f(r.get("cc_half"), "{:.3f}"),
|
|
f(r.get("isa"), "{:.1f}"),
|
|
f(r.get("isa_ref"), "{:.1f}"), pct(r.get("indexing_rate"), 0)]
|
|
+ [f(r.get(k), "{:.3f}") for _, k in model]
|
|
+ ([r.get("model_fit") or "-"] if model else [])
|
|
+ [f(r.get("wall_s"), "{:.0f}"), note])
|
|
doc.table(head, rows, num=range(5, len(head) - 1),
|
|
row_class=lambda row: "fail" if row[2] == "fail" else "",
|
|
cell_class=lambda i, x: {"pass": "pass", "fail": "fail"}.get(x) if i == 2 else None)
|
|
|
|
if baseline:
|
|
bman, bres = load_run(baseline)
|
|
doc.h(2, f"Delta vs baseline {os.path.basename(os.path.normpath(baseline))}")
|
|
doc.p(f"Baseline rugnux {bman['binary'].get('version', '?')}. Pass rates on the sets both runs have:")
|
|
both = {key(r) for r in res} & {key(r) for r in bres}
|
|
rows = []
|
|
for arm in arms:
|
|
ra = [r for r in bres if key(r) in both and r["arm"] == arm]
|
|
rb = [r for r in res if key(r) in both and r["arm"] == arm]
|
|
rows.append([arm, len(rb), rate(ra), rate(rb)])
|
|
doc.table(["arm", "common sets", "baseline", "this run"], rows, num=(1,))
|
|
doc.p("Rows whose result moved beyond noise (thresholds in report.py NOISE; confirm with "
|
|
"`compare --rerun-changed` before believing a single-set change):")
|
|
compare_table(doc, compare_rows(bres, res))
|
|
return doc
|
|
|
|
|
|
def lowres_cell(r):
|
|
"""rugnux / XDS R_meas of the lowest-resolution shell, with the shells' high-resolution limits."""
|
|
return (pct(r.get("refres_lowres_r_meas")) + " / " + pct(r.get("lowres_r_meas_ref"))
|
|
+ f" ({f(r.get('refres_lowres_d'))} / {f(r.get('lowres_d_ref'))})")
|
|
|
|
|
|
def refres_note(r):
|
|
"""How to read a row of the reference-range table."""
|
|
if r.get("refres_shells_past_limit"):
|
|
return (f"coverage: own cut {f(r.get('d_min'))} A is coarser than the reference, "
|
|
f"{r['refres_shells_past_limit']:.0f} shell(s) not merged")
|
|
return "like for like"
|