leonarski_fandClaude Opus 5
e016bc6c78
Data quality - resolution, CC1/2, R_meas, completeness, ISa - is no longer a pass/fail criterion anywhere; it is reported beside the reference in the tables and plots for a human to judge. The CC1/2 noise ratio against XDS stays as a reported guide. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013nW6FNRP1bBJJ8pfHiByAT
489 lines
26 KiB
Python
489 lines
26 KiB
Python
"""The per-run report and the run-to-run comparison."""
|
|
import json
|
|
import os
|
|
import socket
|
|
import statistics
|
|
import sys
|
|
|
|
import score
|
|
from render import Doc, ratio_dots, verdict_bars
|
|
|
|
# A change smaller than these is treated as run-to-run noise. They are a starting point, to be
|
|
# re-calibrated from `compare --rerun-changed`, which measures the noise directly.
|
|
NOISE = {
|
|
"d_min": ("rel", 0.02), # 2% of the resolution limit
|
|
"isa": ("rel", 0.05),
|
|
"r_meas": ("rel", 0.05),
|
|
"cc_half": ("abs", 0.005),
|
|
"completeness": ("abs", 1.0), # percentage points
|
|
"cell_dev_pct": ("abs", 0.2), # percentage points
|
|
"rfree": ("abs", 0.005),
|
|
"refres_r_meas": ("rel", 0.05),
|
|
"refres_isa": ("rel", 0.05),
|
|
"refres_lowres_r_meas": ("rel", 0.05),
|
|
"wall_s": ("rel", 0.25), # and at least 10 s - see beyond_noise()
|
|
}
|
|
|
|
|
|
def run_state(man):
|
|
"""complete, aborted (marked so), running (its runner is alive), or unfinished (killed
|
|
without a trace - treat as aborted)."""
|
|
if man.get("complete"):
|
|
return "complete"
|
|
if man.get("aborted"):
|
|
return "aborted"
|
|
if man.get("pid") and man.get("host") == socket.gethostname():
|
|
try:
|
|
os.kill(man["pid"], 0)
|
|
return "running"
|
|
except OSError:
|
|
pass
|
|
return "unfinished"
|
|
|
|
|
|
def load_run(run_dir, allow_incomplete=False):
|
|
"""A run's manifest and results. A run that did not finish covers an arbitrary part of its
|
|
sets and is refused unless asked for."""
|
|
man = json.load(open(os.path.join(run_dir, "manifest.json")))
|
|
state = run_state(man)
|
|
if state != "complete" and not allow_incomplete:
|
|
sys.exit(f"{run_dir} is {state}, not a finished run; pass --allow-incomplete to use it anyway")
|
|
path = os.path.join(run_dir, "results.json")
|
|
res = json.load(open(path)) if os.path.exists(path) else []
|
|
schema = man.get("results_schema", 1)
|
|
if schema < 2:
|
|
schema1(man, res)
|
|
if schema < 3:
|
|
res[:] = schema2(res)
|
|
else:
|
|
rescore(run_dir, man, res)
|
|
return man, res
|
|
|
|
|
|
def rescore(run_dir, man, res):
|
|
"""Score every row again from its saved report, with today's scorer and today's manifest rows
|
|
(references, `unscored`, `expect`), so a report or a compare never mixes two scorings or two
|
|
references. results.json keeps the verdicts as they were scored when the run finished.
|
|
|
|
A row whose report has no lattice keeps its verdict: why it failed (crash, reader, timeout)
|
|
came from the run, not from the report."""
|
|
own = {(e["arm"], e["id"]): e for e in man["sets"]}
|
|
for r in res:
|
|
e = MANIFESTS.get(r["arm"], {}).get(key(r)[1]) or own.get((r["arm"], r["set"]))
|
|
rep = score.read_report(os.path.join(run_dir, "work", r["arm"], r["set"], "p_report.txt"))
|
|
lattice = rep.get("SPACE_GROUP_NUMBER") and rep.get("UNIT_CELL_CONSTANTS")
|
|
if e and (lattice or e.get("expect") == "no_lattice"):
|
|
r.update(score.judge(dict(e, arm=r["arm"]), rep, ""))
|
|
|
|
|
|
def schema1(man, res):
|
|
"""Read a run from before the variants (results schema 1) as one: it ran each set once, plain
|
|
on the open arm and with XDS's settings on the XDS arms - unless --unforced, which left the
|
|
range free but still passed -A, and is read as `bare`. Its R-factors came from the REFMAC check."""
|
|
for r in res:
|
|
r.setdefault("variant", "xds" if r["arm"] != "open" and man["options"].get("xds_range") else "bare")
|
|
if "model_status" in r:
|
|
for k in ("rfree", "rwork", "rfree_depflags", "rfree_depdata", "rfree_ratio"):
|
|
r["refmac_" + k] = r.pop(k, None)
|
|
r["refmac_status"], r["refmac_reason"] = r.pop("model_status"), r.pop("model_reason", None)
|
|
r["rfree_deposited"] = None
|
|
|
|
|
|
def schema2(res):
|
|
"""Read a run with variants (schema 2) as one run per set: its `bare` row, with the open arm's
|
|
`model` row's R-factors folded in. That is today's processing on the open arm; on the XDS arms
|
|
today's command adds -A where XDS was anomalous. The `xds` rows (XDS's range forced on the
|
|
processing) have no counterpart and are left out."""
|
|
model = {r["set"]: r for r in res if r["variant"] == "model"}
|
|
out = []
|
|
for r in res:
|
|
if r["variant"] != "bare":
|
|
continue
|
|
m = model.get(r["set"])
|
|
if m:
|
|
r.update({k: m.get(k) for k in ("rfree", "rwork", "model_fit", "rfree_deposited", "rfree_ratio")})
|
|
out.append(r)
|
|
return out
|
|
|
|
|
|
def expected_rows(man):
|
|
"""Rows a finished run has: one per set (per set and variant before schema 3)."""
|
|
if man.get("results_schema", 1) >= 3:
|
|
return len(man["sets"])
|
|
return sum(len(man.get("variants", {}).get(e["arm"], [None])) for e in man["sets"])
|
|
|
|
|
|
def check_baseline(baseline, private):
|
|
"""A baseline must have finished, and be as private as the run it is compared with: the delta
|
|
table names its sets."""
|
|
man, _ = load_run(baseline)
|
|
if bool(man.get("private")) != bool(private):
|
|
sys.exit(f"baseline {baseline} is {'' if man.get('private') else 'not '}a private run; "
|
|
"a report may only be compared with a run of its own kind")
|
|
|
|
|
|
# Version of the results.json row schema (README.md, "results.json"). Bump it when a field changes
|
|
# meaning or is removed; adding a field does not need it.
|
|
# 2: one row per set AND variant (`variant`, `first_read`); `rfree`/`rwork`/`rfree_ratio` are rugnux's
|
|
# own --model R-factors, the REFMAC check's moved to `refmac_*`.
|
|
# 3: one row per set again (one command per arm, no `variant`/`first_read`); `refres_*` added;
|
|
# `sgno`/`sg` are the data's own group where --model relabelled the hand (`sg_label`).
|
|
RESULTS_SCHEMA = 3
|
|
|
|
# {arm: {old set id: current id}} from the manifests (battery.py sets it), so a run made before a
|
|
# dataset was renamed still pairs with one made after
|
|
ALIASES = {}
|
|
|
|
# {arm: {set id: manifest row}} of today's manifests (battery.py sets it); rescore() scores with them
|
|
MANIFESTS = {}
|
|
|
|
|
|
def key(r):
|
|
return (r["arm"], ALIASES.get(r["arm"], {}).get(r["set"], r["set"]))
|
|
|
|
|
|
def f(x, fmt="{:.2f}", dash="-"):
|
|
return dash if x is None else fmt.format(x)
|
|
|
|
|
|
def pct(x, nd=1):
|
|
return "-" if x is None else f"{100 * x:.{nd}f}%"
|
|
|
|
|
|
def median(xs):
|
|
xs = [x for x in xs if x is not None]
|
|
return statistics.median(xs) if xs else None
|
|
|
|
|
|
def quartiles(xs):
|
|
xs = sorted(x for x in xs if x is not None)
|
|
if not xs:
|
|
return None
|
|
if len(xs) < 4:
|
|
return xs[0], xs[len(xs) // 2], xs[len(xs) // 2], xs[len(xs) // 2], xs[-1]
|
|
q = statistics.quantiles(xs, n=4)
|
|
return xs[0], q[0], q[1], q[2], xs[-1]
|
|
|
|
|
|
def beyond_noise(name, a, b):
|
|
if a is None or b is None:
|
|
return (a is None) != (b is None)
|
|
kind, tol = NOISE[name]
|
|
d = abs(b - a)
|
|
if name == "wall_s" and d < 10:
|
|
return False
|
|
return d > (tol * abs(a) if kind == "rel" else tol)
|
|
|
|
|
|
def changes(ra, rb):
|
|
"""What changed between two results of the same set, beyond noise."""
|
|
out = []
|
|
if ra["verdict"] != rb["verdict"]:
|
|
out.append(f"verdict {ra['verdict']}->{rb['verdict']}")
|
|
if ra.get("sgno") != rb.get("sgno"):
|
|
out.append("space group")
|
|
if ra.get("volume_ratio") and rb.get("volume_ratio") and \
|
|
abs(ra["volume_ratio"] - rb["volume_ratio"]) > 0.05:
|
|
out.append("lattice")
|
|
for name in ("d_min", "isa", "r_meas", "cc_half", "completeness", "cell_dev_pct", "rfree"):
|
|
if beyond_noise(name, ra.get(name), rb.get(name)):
|
|
out.append(name)
|
|
# the like-for-like table only where both runs have one (a pre-schema-3 run has none)
|
|
for name in ("refres_r_meas", "refres_isa", "refres_lowres_r_meas"):
|
|
if ra.get(name) is not None and rb.get(name) is not None and beyond_noise(name, ra[name], rb[name]):
|
|
out.append(name)
|
|
if beyond_noise("wall_s", ra.get("wall_s"), rb.get("wall_s")):
|
|
out.append("time")
|
|
return out
|
|
|
|
|
|
def compare_rows(res_a, res_b):
|
|
"""[(key, ra, rb, changes)] over the sets both runs have, then those only one has."""
|
|
a = {key(r): r for r in res_a}
|
|
b = {key(r): r for r in res_b}
|
|
rows = [(k, a[k], b[k], changes(a[k], b[k])) for k in sorted(a) if k in b]
|
|
rows += [(k, a[k], None, ["only in A"]) for k in sorted(a) if k not in b]
|
|
rows += [(k, None, b[k], ["only in B"]) for k in sorted(b) if k not in a]
|
|
return rows
|
|
|
|
|
|
def arrow(x, y, fmt=lambda v: str(v)):
|
|
x = "-" if x is None else fmt(x)
|
|
y = "-" if y is None else fmt(y)
|
|
return x if x == y else f"{x} -> {y}"
|
|
|
|
|
|
def compare_table(doc, rows, only_changed=True):
|
|
head = ["set", "arm", "verdict", "space group", "d_min", "ISa", "R_meas", "CC1/2",
|
|
"cell dev %", "R_free", "ref-range R_meas", "ref-range low-res R_meas", "time s", "beyond noise"]
|
|
out = []
|
|
for (arm, s), ra, rb, ch in rows:
|
|
if only_changed and not ch:
|
|
continue
|
|
ra, rb = ra or {}, rb or {}
|
|
out.append([s, arm, arrow(ra.get("verdict"), rb.get("verdict")),
|
|
arrow(ra.get("sg"), rb.get("sg")),
|
|
arrow(ra.get("d_min"), rb.get("d_min"), lambda v: f"{v:.2f}"),
|
|
arrow(ra.get("isa"), rb.get("isa"), lambda v: f"{v:.1f}"),
|
|
arrow(ra.get("r_meas"), rb.get("r_meas"), lambda v: f"{100 * v:.1f}%"),
|
|
arrow(ra.get("cc_half"), rb.get("cc_half"), lambda v: f"{v:.3f}"),
|
|
arrow(ra.get("cell_dev_pct"), rb.get("cell_dev_pct"), lambda v: f"{v:.2f}"),
|
|
arrow(ra.get("rfree"), rb.get("rfree"), lambda v: f"{v:.3f}"),
|
|
arrow(ra.get("refres_r_meas"), rb.get("refres_r_meas"), lambda v: f"{100 * v:.1f}%"),
|
|
arrow(ra.get("refres_lowres_r_meas"), rb.get("refres_lowres_r_meas"),
|
|
lambda v: f"{100 * v:.1f}%"),
|
|
arrow(ra.get("wall_s"), rb.get("wall_s"), lambda v: f"{v:.0f}"),
|
|
", ".join(ch)])
|
|
doc.table(head, out, row_class=lambda r: "fail" if "verdict" in r[-1] else "")
|
|
|
|
|
|
def crystals(rs):
|
|
"""Rows that describe a crystal - the no-crystal controls would only pollute a distribution."""
|
|
return [r for r in rs if "control" not in r.get("tags", [])]
|
|
|
|
|
|
def scored(rs):
|
|
return [r for r in rs if r["verdict"] in ("pass", "fail")]
|
|
|
|
|
|
def rate(rs):
|
|
s = scored(rs)
|
|
n = sum(r["verdict"] == "pass" for r in s)
|
|
return f"{n}/{len(s)} ({100 * n / len(s):.0f}%)" if s else "-"
|
|
|
|
|
|
def build(run_dir, baseline=None, allow_incomplete=False):
|
|
"""Markdown and HTML report of one run, with the delta against a baseline run if given."""
|
|
man, res = load_run(run_dir, allow_incomplete)
|
|
if baseline:
|
|
check_baseline(baseline, man.get("private"))
|
|
b = man["binary"]
|
|
doc = Doc(f"Rugnux battery - {os.path.basename(os.path.normpath(run_dir))}")
|
|
if man.get("private"):
|
|
doc.p("PRIVATE ARM - confidential user data. This report stays in the run directory; it "
|
|
"must not be copied into the repository, an issue or any public report.")
|
|
state = run_state(man)
|
|
if state != "complete":
|
|
why = f" ({man['aborted']['reason']})" if man.get("aborted") else ""
|
|
doc.p(f"RUN {state.upper()}{why}: {len(res)} of {expected_rows(man)} rows have a "
|
|
"result. Not a reference and not comparable with a finished run.")
|
|
if man.get("results_schema", 1) < 3:
|
|
doc.p("A run from before the one-command battery: shown by its `bare` rows (with the `model` "
|
|
"rows' R-factors on the open arm); its `xds` rows have no counterpart and are left out.")
|
|
doc.ul([
|
|
f"rugnux {b.get('version', '?')}, sha256 {b['sha256'][:16]}, build flags: {b.get('flags') or 'unknown'}",
|
|
f"runner {man['runner_git']}, host {man['host']}, {man['start']} -> {man.get('end') or 'NOT FINISHED'}",
|
|
f"arms: {', '.join(man['arms'])}; {len(man['sets'])} sets"
|
|
+ (f" (subset: {man['subset']})" if man.get("subset") else " (full arm)"),
|
|
"timing: NOT a reference - the GPU was shared" if man.get("gpu_shared")
|
|
else "timing: GPU was not shared during the run",
|
|
f"command options: {' '.join(man['options']['extra_args']) or '(none beyond output selection)'}",
|
|
] + [f"command, {arm} arm: {d}" for arm, d in man.get("command_doc", {}).items()])
|
|
|
|
arms = [arm for arm in man["arms"] if any(r["arm"] == arm for r in res)]
|
|
order = {arm: i for i, arm in enumerate(arms)}
|
|
|
|
doc.h(2, "Summary")
|
|
head = ["arm", "sets", "pass", "fail", "unscored", "not run", "pass rate",
|
|
"median res. gain", "median ISa", "median R_meas", "median time s", "total time min"]
|
|
rows = []
|
|
for arm in arms:
|
|
rs = [r for r in res if r["arm"] == arm]
|
|
cnt = {x: sum(r["verdict"] == x for r in rs) for x in ("pass", "fail", "unscored", "not_run")}
|
|
xs = crystals(rs)
|
|
rows.append([arm, len(rs), cnt["pass"], cnt["fail"], cnt["unscored"], cnt["not_run"], rate(rs),
|
|
f(median(r.get("res_gain_pct") for r in xs), "{:+.1f}%"),
|
|
f(median(r.get("isa") for r in xs), "{:.1f}"),
|
|
pct(median(r.get("r_meas") for r in xs)),
|
|
f(median(r.get("wall_s") for r in rs), "{:.0f}"),
|
|
f(sum(r.get("wall_s") or 0 for r in rs) / 60, "{:.1f}")])
|
|
doc.table(head, rows, num=range(1, 12))
|
|
doc.svg(verdict_bars([(arm, {x: sum(r["verdict"] == x and r["arm"] == arm for r in res)
|
|
for x in ("pass", "fail", "unscored", "not_run")})
|
|
for arm in arms]), "verdicts per arm")
|
|
doc.p("Pass rate is over the scored sets (pass + fail). Every number in this table is rugnux's own "
|
|
"run at its own resolution cut: what a user gets. Resolution gain is (reference d_min - our "
|
|
"d_min) / reference d_min: positive = finer than the deposition, or than XDS. Medians leave "
|
|
"out the no-crystal controls. R_meas and CC1/2 are pooled over each run's own resolution "
|
|
"range, so they describe a run and do not rank two; the like-for-like comparison with XDS "
|
|
"is the reference-range table below. Each set runs once, so every time includes reading "
|
|
"its images from disk (unless they were still in the page cache from before the run).")
|
|
|
|
doc.h(3, "By population")
|
|
rows = []
|
|
for arm in arms:
|
|
rs = [r for r in res if r["arm"] == arm]
|
|
for tag in sorted({t for r in rs for t in r.get("tags", [])}):
|
|
sub = [r for r in rs if tag in r.get("tags", [])]
|
|
rows.append([arm, tag, len(sub), rate(sub), f(median(r.get("res_gain_pct") for r in sub), "{:+.1f}%")])
|
|
doc.table(["arm", "population", "sets", "pass rate", "median res. gain"], rows, num=(2, 4))
|
|
|
|
doc.h(3, "Distributions (sets that merged, controls left out)")
|
|
rows = []
|
|
for arm in arms:
|
|
rs = crystals(r for r in res if r["arm"] == arm)
|
|
for name, lab, fm in (("isa", "ISa", "{:.1f}"), ("r_meas", "R_meas", None),
|
|
("cc_half", "CC1/2", "{:.3f}"), ("res_gain_pct", "res. gain %", "{:+.1f}"),
|
|
("isa_ratio", "ref-range ISa / XDS", "{:.3f}"),
|
|
("r_meas_ratio", "ref-range R_meas / XDS", "{:.3f}"),
|
|
("lowres_r_meas", "low-res shell R_meas", None),
|
|
("lowres_r_meas_ratio", "ref-range low-res R_meas / XDS", "{:.3f}"),
|
|
("cc_half_noise_ratio", "ref-range CC1/2 noise / XDS", "{:.2f}"),
|
|
("rfree", "R_free (placement)", "{:.3f}"),
|
|
("rfree_ratio", "R_free ratio", "{:.3f}"), ("wall_s", "time s", "{:.0f}")):
|
|
q = quartiles(r.get(name) for r in rs)
|
|
if q:
|
|
cells = [pct(x) if fm is None else f(x, fm) for x in q]
|
|
rows.append([arm, lab, sum(r.get(name) is not None for r in rs)] + cells)
|
|
doc.table(["arm", "metric", "n", "min", "q1", "median", "q3", "max"], rows, num=range(2, 8))
|
|
|
|
doc.h(3, "Per set, against the reference")
|
|
for arm in arms:
|
|
pts = [(r["set"], r["d_min"] / r["d_min_ref"], r["verdict"],
|
|
f"d_min {r['d_min']:.2f} vs {r['d_min_ref']:.2f} A, {r['verdict']}")
|
|
for r in res if r["arm"] == arm and r.get("d_min") and r.get("d_min_ref")]
|
|
if pts:
|
|
doc.svg(ratio_dots(pts, f"{arm}: d_min ratio"),
|
|
f"{arm}: d_min(rugnux, own cut) / d_min(reference), one point per set, sorted; below 1 "
|
|
"= finer than the reference")
|
|
for name, what, cap in (
|
|
("isa_ratio", "ISa", "ISa of the reference-range table (the error model refitted on it) / "
|
|
"XDS's ISa; above 1 = rugnux's error model is the better one"),
|
|
("r_meas_ratio", "R_meas", "R_meas over the reference range / XDS's R_meas; below 1 = "
|
|
"rugnux's merge is the more consistent one"),
|
|
("lowres_r_meas_ratio", "low-res R_meas", "R_meas of the lowest-resolution shell of the "
|
|
"reference-range table / of XDS's table; below 1 = "
|
|
"rugnux's strong reflections agree better"),
|
|
("cc_half_noise_ratio", "CC1/2 noise", "half-set noise over the reference range read off CC1/2 "
|
|
"(1 / CC1/2 - 1) / XDS's; 2 = as noisy as XDS's merge "
|
|
"would be with half its observations (not scored)")):
|
|
for arm in arms:
|
|
pts = [(r["set"], r[name], r["verdict"], f"{what} ratio, {refres_note(r)}")
|
|
for r in res if r["arm"] == arm and r.get(name)]
|
|
if pts:
|
|
doc.svg(ratio_dots(pts, f"{arm}: reference-range {what} ratio"),
|
|
f"{arm}: {cap}. One point per set, sorted.")
|
|
pts = [(r["set"], r["rfree_ratio"], r["verdict"],
|
|
f"R_free {r['rfree']:.3f} vs deposited {r['rfree_deposited']:.3f}")
|
|
for r in res if r.get("rfree_ratio")]
|
|
if pts:
|
|
doc.svg(ratio_dots(pts, "R_free ratio"),
|
|
"R_free of the deposited model against our merge, as rugnux reports it with --model (a "
|
|
"rigid-body placement, scored on rugnux's own free set: a trend number, not a refinement) / "
|
|
"the R_free the depositor published, one point per set, sorted; below 1 = lower than published")
|
|
pts = [(r["set"], r["refmac_rfree_ratio"], r["verdict"],
|
|
f"R_free {r['refmac_rfree_depflags']:.3f} vs {r['refmac_rfree_depdata']:.3f} on the deposited data")
|
|
for r in res if r.get("refmac_rfree_ratio")]
|
|
if pts:
|
|
doc.svg(ratio_dots(pts, "REFMAC R_free ratio"),
|
|
"REFMAC check (--model-check): R_free of the deposited model against our merge / against "
|
|
"the deposited structure factors, both on the depositor's free set with the same protocol, "
|
|
"one point per set, sorted; below 1 = our data fit the model better")
|
|
|
|
xds = [r for r in res if r.get("refres_range")]
|
|
if xds:
|
|
doc.h(3, "Like for like with XDS: the reference-range table")
|
|
doc.p("The same merge, binned a second time over XDS's range (--report-resolution; report-only, "
|
|
"nothing was processed differently for it), against XDS's CORRECT.LP totals. Where "
|
|
"rugnux's own cut is coarser than the reference ('coverage' in the last column), the "
|
|
"shells past it are not merged at all: the completeness there is coverage of XDS's "
|
|
"range, not a quality loss, and the other rugnux numbers are over the shells it reached. "
|
|
"XDS's totals are over its own merged range, which is the reference range except where "
|
|
"the reference d_min was derived (see below).")
|
|
rows = []
|
|
for r in sorted(xds, key=lambda r: (order.get(r["arm"], 9), r["set"])):
|
|
rows.append([r["set"], r["arm"], r["refres_range"], f(r.get("d_min")),
|
|
f(r.get("refres_completeness"), "{:.1f}") + " / " + f(r.get("completeness_ref"), "{:.1f}"),
|
|
f(r.get("refres_multiplicity"), "{:.1f}") + " / " + f(r.get("multiplicity_ref"), "{:.1f}"),
|
|
f(r.get("refres_i_over_sigma"), "{:.1f}"),
|
|
pct(r.get("refres_r_meas")) + " / " + pct(r.get("r_meas_ref")),
|
|
lowres_cell(r),
|
|
f(r.get("refres_cc_half"), "{:.4f}") + " / " + f(r.get("cc_half_ref"), "{:.3f}"),
|
|
f(r.get("cc_half_noise_ratio"), "{:.2f}"),
|
|
f(r.get("refres_isa"), "{:.1f}") + " / " + f(r.get("isa_ref"), "{:.1f}"),
|
|
refres_note(r)])
|
|
doc.table(["set", "arm", "range A", "own d_min", "compl % (rugnux / XDS)", "mult", "I/sigma",
|
|
"R_meas", "low-res R_meas (to d A)", "CC1/2", "CC1/2 noise ratio", "ISa", "reading"],
|
|
rows, num=(3, 6, 10))
|
|
doc.p("Data quality is not scored; this table is for a human to judge. "
|
|
"CC1/2 = S / (S + E) (signal and half-set error variances), so 1 / CC1/2 - 1 = E / S is the "
|
|
"half-set noise; the noise ratio is rugnux's over XDS's, with XDS's CC1/2 taken at the "
|
|
"bottom of its printed rounding (-0.0005); 2 would mean noisier than XDS's merge with half "
|
|
"its observations. The low-resolution R_meas is the lowest shell of each table (the shells "
|
|
"are XDS's, so both cover the same reflections); reported like everything here.")
|
|
|
|
derived = {r["set"]: r for r in res if (r.get("d_min_ref_rule") or "xds_range") != "xds_range"}
|
|
if derived:
|
|
doc.p("XDS merged past its own signal (CC1/2 of its finest shell not significant), so the "
|
|
"reference d_min is where XDS's CC1/2 falls through 0.30, and that is also the d_min of the "
|
|
"reference-range table: "
|
|
+ "; ".join(f"{s} (XDS range {f(r.get('d_min_xds'))} A, reference "
|
|
f"{f(r.get('d_min_ref'))} A)" for s, r in sorted(derived.items()))
|
|
+ ". XDS's pooled R_meas, CC1/2 and completeness are over its whole range.")
|
|
|
|
doc.h(3, "Failures")
|
|
fails = [r for r in res if r["verdict"] == "fail"]
|
|
doc.table(["set", "arm", "cause", "reason"],
|
|
[[r["set"], r["arm"], r["cause"], r["reason"]] for r in fails])
|
|
other = [r for r in res if r["verdict"] in ("unscored", "not_run")]
|
|
if other:
|
|
doc.p("Not scored: " + "; ".join(f"{r['set']} ({r['reason']})" for r in other))
|
|
|
|
doc.h(3, "Per set")
|
|
doc.p("Space group: on the open arm the data's own determination. Where --model put the model's "
|
|
"enantiomorph on the label, the label is in the note and the group scored is the search's.")
|
|
model = [("R_free", "rfree"), ("R_work", "rwork"), ("CC model", "cc_model"), ("dep R_free", "rfree_deposited"),
|
|
("R_free ratio", "rfree_ratio")] if any(r.get("rfree") for r in res) else []
|
|
model += [("REFMAC R_free", "refmac_rfree_depflags"), ("REFMAC dep data", "refmac_rfree_depdata"),
|
|
("REFMAC ratio", "refmac_rfree_ratio")] if any(r.get("refmac_status") for r in res) else []
|
|
head = ["set", "arm", "verdict", "space group", "ref", "cell dev %", "V ratio", "d_min", "ref d_min",
|
|
"gain", "compl %", "mult", "R_meas", "low-res R_meas", "CC1/2", "ISa", "ref ISa", "idx"] + \
|
|
[h for h, _ in model] + (["model fit"] if model else []) + ["time s", "note"]
|
|
rows = []
|
|
for r in sorted(res, key=lambda r: (order.get(r["arm"], 9), r["set"])):
|
|
note = "; ".join(x for x in (r.get("reason"), r.get("sg_label") and f"labelled {r['sg_label']} from the model")
|
|
if x)
|
|
rows.append([r["set"], r["arm"], r["verdict"], r.get("sg") or "-", r.get("sg_ref") or "-",
|
|
f(r.get("cell_dev_pct")), f(r.get("volume_ratio"), "{:.3f}"),
|
|
f(r.get("d_min")), f(r.get("d_min_ref")), f(r.get("res_gain_pct"), "{:+.1f}%"),
|
|
f(r.get("completeness"), "{:.1f}"), f(r.get("multiplicity"), "{:.1f}"),
|
|
pct(r.get("r_meas")), pct(r.get("lowres_r_meas")), f(r.get("cc_half"), "{:.3f}"),
|
|
f(r.get("isa"), "{:.1f}"),
|
|
f(r.get("isa_ref"), "{:.1f}"), pct(r.get("indexing_rate"), 0)]
|
|
+ [f(r.get(k), "{:.3f}") for _, k in model]
|
|
+ ([r.get("model_fit") or "-"] if model else [])
|
|
+ [f(r.get("wall_s"), "{:.0f}"), note])
|
|
doc.table(head, rows, num=range(5, len(head) - 1),
|
|
row_class=lambda row: "fail" if row[2] == "fail" else "",
|
|
cell_class=lambda i, x: {"pass": "pass", "fail": "fail"}.get(x) if i == 2 else None)
|
|
|
|
if baseline:
|
|
bman, bres = load_run(baseline)
|
|
doc.h(2, f"Delta vs baseline {os.path.basename(os.path.normpath(baseline))}")
|
|
doc.p(f"Baseline rugnux {bman['binary'].get('version', '?')}. Pass rates on the sets both runs have:")
|
|
both = {key(r) for r in res} & {key(r) for r in bres}
|
|
rows = []
|
|
for arm in arms:
|
|
ra = [r for r in bres if key(r) in both and r["arm"] == arm]
|
|
rb = [r for r in res if key(r) in both and r["arm"] == arm]
|
|
rows.append([arm, len(rb), rate(ra), rate(rb)])
|
|
doc.table(["arm", "common sets", "baseline", "this run"], rows, num=(1,))
|
|
doc.p("Rows whose result moved beyond noise (thresholds in report.py NOISE; confirm with "
|
|
"`compare --rerun-changed` before believing a single-set change):")
|
|
compare_table(doc, compare_rows(bres, res))
|
|
return doc
|
|
|
|
|
|
def lowres_cell(r):
|
|
"""rugnux / XDS R_meas of the lowest-resolution shell, with the shells' high-resolution limits."""
|
|
return (pct(r.get("refres_lowres_r_meas")) + " / " + pct(r.get("lowres_r_meas_ref"))
|
|
+ f" ({f(r.get('refres_lowres_d'))} / {f(r.get('lowres_d_ref'))})")
|
|
|
|
|
|
def refres_note(r):
|
|
"""How to read a row of the reference-range table."""
|
|
if r.get("refres_shells_past_limit"):
|
|
return (f"coverage: own cut {f(r.get('d_min'))} A is coarser than the reference, "
|
|
f"{r['refres_shells_past_limit']:.0f} shell(s) not merged")
|
|
return "like for like"
|