"""The per-run report and the run-to-run comparison.""" import json import multiprocessing import os import socket import statistics import sys import depdata_check import score from render import Doc, ratio_dots, verdict_bars # A change smaller than these is treated as run-to-run noise. They are a starting point, to be # re-calibrated from `compare --rerun-changed`, which measures the noise directly. NOISE = { "d_min": ("rel", 0.02), # 2% of the resolution limit "isa": ("rel", 0.05), "r_meas": ("rel", 0.05), "cc_half": ("abs", 0.005), "completeness": ("abs", 1.0), # percentage points "cell_dev_pct": ("abs", 0.2), # percentage points "rfree": ("abs", 0.005), "rmodel_shell_scaled": ("abs", 0.005), "radial_misfit": ("abs", 0.05), "refres_r_meas": ("rel", 0.05), "refres_isa": ("rel", 0.05), "refres_lowres_r_meas": ("rel", 0.05), "wall_s": ("rel", 0.25), # and at least 10 s - see beyond_noise() } def run_state(man): """complete, aborted (marked so), running (its runner is alive), or unfinished (killed without a trace - treat as aborted).""" if man.get("complete"): return "complete" if man.get("aborted"): return "aborted" if man.get("pid") and man.get("host") == socket.gethostname(): try: os.kill(man["pid"], 0) return "running" except OSError: pass return "unfinished" def load_run(run_dir, allow_incomplete=False): """A run's manifest and results. A run that did not finish covers an arbitrary part of its sets and is refused unless asked for.""" man = json.load(open(os.path.join(run_dir, "manifest.json"))) state = run_state(man) if state != "complete" and not allow_incomplete: sys.exit(f"{run_dir} is {state}, not a finished run; pass --allow-incomplete to use it anyway") path = os.path.join(run_dir, "results.json") res = json.load(open(path)) if os.path.exists(path) else [] schema = man.get("results_schema", 1) if schema < 2: schema1(man, res) if schema < 3: res[:] = schema2(res) else: rescore(run_dir, man, res) return man, res def rescore(run_dir, man, res): """Score every row again from its saved report, with today's scorer and today's manifest rows (references, `unscored`, `expect`), so a report or a compare never mixes two scorings or two references. results.json keeps the verdicts as they were scored when the run finished. A row whose report has no lattice keeps its verdict: why it failed (crash, reader, timeout) came from the run, not from the report.""" own = {(e["arm"], e["id"]): e for e in man["sets"]} for r in res: e = MANIFESTS.get(r["arm"], {}).get(key(r)[1]) or own.get((r["arm"], r["set"])) rep = score.read_report(os.path.join(run_dir, "work", r["arm"], r["set"], "p_report.txt")) lattice = rep.get("SPACE_GROUP_NUMBER") and rep.get("UNIT_CELL_CONSTANTS") if e and (lattice or e.get("expect") == "no_lattice"): r.update(score.judge(dict(e, arm=r["arm"]), rep, "")) def _depdata(job): os.nice(10) return depdata_check.check_set(*job) def fill_depdata(run_dir, res): """The depositor-data comparison (depdata_check.py) for open-arm rows of a run made before it existed, computed from the saved merge, four sets at a time.""" todo = [r for r in res if r["arm"] == "open" and "dep_status" not in r and r.get("model") and os.path.exists(os.path.join(run_dir, "work", "open", r["set"], "p.mtz"))] if not todo: return print(f"depositor-data comparison for {len(todo)} sets of an older run ...", file=sys.stderr) jobs = [(os.path.join(run_dir, "work", "open", r["set"]), r["set"], PDB_CACHE or depdata_check.model_check.DEFAULT_CACHE) for r in todo] with multiprocessing.Pool(4) as pool: for r, out in zip(todo, pool.map(_depdata, jobs, chunksize=1)): r.update(out) def schema1(man, res): """Read a run from before the variants (results schema 1) as one: it ran each set once, plain on the open arm and with XDS's settings on the XDS arms - unless --unforced, which left the range free but still passed -A, and is read as `bare`. Its R-factors came from the REFMAC check.""" for r in res: r.setdefault("variant", "xds" if r["arm"] != "open" and man["options"].get("xds_range") else "bare") if "model_status" in r: for k in ("rfree", "rwork", "rfree_depflags", "rfree_depdata", "rfree_ratio"): r["refmac_" + k] = r.pop(k, None) r["refmac_status"], r["refmac_reason"] = r.pop("model_status"), r.pop("model_reason", None) r["rfree_deposited"] = None def schema2(res): """Read a run with variants (schema 2) as one run per set: its `bare` row, with the open arm's `model` row's R-factors folded in. That is today's processing on the open arm; on the XDS arms today's command adds -A where XDS was anomalous. The `xds` rows (XDS's range forced on the processing) have no counterpart and are left out.""" model = {r["set"]: r for r in res if r["variant"] == "model"} out = [] for r in res: if r["variant"] != "bare": continue m = model.get(r["set"]) if m: r.update({k: m.get(k) for k in ("rfree", "rwork", "model_fit", "rfree_deposited", "rfree_ratio")}) out.append(r) return out def expected_rows(man): """Rows a finished run has: one per set (per set and variant before schema 3).""" if man.get("results_schema", 1) >= 3: return len(man["sets"]) return sum(len(man.get("variants", {}).get(e["arm"], [None])) for e in man["sets"]) def check_baseline(baseline, private): """A baseline must have finished, and be as private as the run it is compared with: the delta table names its sets.""" man, _ = load_run(baseline) if bool(man.get("private")) != bool(private): sys.exit(f"baseline {baseline} is {'' if man.get('private') else 'not '}a private run; " "a report may only be compared with a run of its own kind") # Version of the results.json row schema (README.md, "results.json"). Bump it when a field changes # meaning or is removed; adding a field does not need it. # 2: one row per set AND variant (`variant`, `first_read`); `rfree`/`rwork`/`rfree_ratio` are rugnux's # own --model R-factors, the REFMAC check's moved to `refmac_*`. # 3: one row per set again (one command per arm, no `variant`/`first_read`); `refres_*` added; # `sgno`/`sg` are the data's own group where --model relabelled the hand (`sg_label`). RESULTS_SCHEMA = 3 # {arm: {old set id: current id}} from the manifests (battery.py sets it), so a run made before a # dataset was renamed still pairs with one made after ALIASES = {} # {arm: {set id: manifest row}} of today's manifests (battery.py sets it); rescore() scores with them MANIFESTS = {} # the site's cache of deposited models and structure factors (battery.py sets it) PDB_CACHE = None def key(r): return (r["arm"], ALIASES.get(r["arm"], {}).get(r["set"], r["set"])) def f(x, fmt="{:.2f}", dash="-"): return dash if x is None else fmt.format(x) def pct(x, nd=1): return "-" if x is None else f"{100 * x:.{nd}f}%" def median(xs): xs = [x for x in xs if x is not None] return statistics.median(xs) if xs else None def quartiles(xs): xs = sorted(x for x in xs if x is not None) if not xs: return None if len(xs) < 4: return xs[0], xs[len(xs) // 2], xs[len(xs) // 2], xs[len(xs) // 2], xs[-1] q = statistics.quantiles(xs, n=4) return xs[0], q[0], q[1], q[2], xs[-1] def beyond_noise(name, a, b): if a is None or b is None: return (a is None) != (b is None) kind, tol = NOISE[name] d = abs(b - a) if name == "wall_s" and d < 10: return False return d > (tol * abs(a) if kind == "rel" else tol) def changes(ra, rb): """What changed between two results of the same set, beyond noise.""" out = [] if ra["verdict"] != rb["verdict"]: out.append(f"verdict {ra['verdict']}->{rb['verdict']}") if ra.get("sgno") != rb.get("sgno"): out.append("space group") if ra.get("volume_ratio") and rb.get("volume_ratio") and \ abs(ra["volume_ratio"] - rb["volume_ratio"]) > 0.05: out.append("lattice") for name in ("d_min", "isa", "r_meas", "cc_half", "completeness", "cell_dev_pct"): if beyond_noise(name, ra.get(name), rb.get(name)): out.append(name) # The R that is comparable between runs, and the radial profile that decides whether `rfree` # beside it can be read at all (both only where both runs carry them). for name in ("rmodel_shell_scaled", "radial_misfit", "rfree"): if ra.get(name) is not None and rb.get(name) is not None and beyond_noise(name, ra[name], rb[name]): out.append(name) # the like-for-like table only where both runs have one (a pre-schema-3 run has none) for name in ("refres_r_meas", "refres_isa", "refres_lowres_r_meas"): if ra.get(name) is not None and rb.get(name) is not None and beyond_noise(name, ra[name], rb[name]): out.append(name) if beyond_noise("wall_s", ra.get("wall_s"), rb.get("wall_s")): out.append("time") return out def compare_rows(res_a, res_b): """[(key, ra, rb, changes)] over the sets both runs have, then those only one has.""" a = {key(r): r for r in res_a} b = {key(r): r for r in res_b} rows = [(k, a[k], b[k], changes(a[k], b[k])) for k in sorted(a) if k in b] rows += [(k, a[k], None, ["only in A"]) for k in sorted(a) if k not in b] rows += [(k, None, b[k], ["only in B"]) for k in sorted(b) if k not in a] return rows def arrow(x, y, fmt=lambda v: str(v)): x = "-" if x is None else fmt(x) y = "-" if y is None else fmt(y) return x if x == y else f"{x} -> {y}" def compare_table(doc, rows, only_changed=True): head = ["set", "arm", "verdict", "space group", "d_min", "ISa", "R_meas", "CC1/2", "cell dev %", "R_model (shell-scaled)", "R_free", "radial misfit", "ref-range R_meas", "ref-range low-res R_meas", "time s", "beyond noise"] out = [] for (arm, s), ra, rb, ch in rows: if only_changed and not ch: continue ra, rb = ra or {}, rb or {} out.append([s, arm, arrow(ra.get("verdict"), rb.get("verdict")), arrow(ra.get("sg"), rb.get("sg")), arrow(ra.get("d_min"), rb.get("d_min"), lambda v: f"{v:.2f}"), arrow(ra.get("isa"), rb.get("isa"), lambda v: f"{v:.1f}"), arrow(ra.get("r_meas"), rb.get("r_meas"), lambda v: f"{100 * v:.1f}%"), arrow(ra.get("cc_half"), rb.get("cc_half"), lambda v: f"{v:.3f}"), arrow(ra.get("cell_dev_pct"), rb.get("cell_dev_pct"), lambda v: f"{v:.2f}"), arrow(ra.get("rmodel_shell_scaled"), rb.get("rmodel_shell_scaled"), lambda v: f"{v:.3f}"), arrow(ra.get("rfree"), rb.get("rfree"), lambda v: f"{v:.3f}"), arrow(ra.get("radial_misfit"), rb.get("radial_misfit"), lambda v: f"{v:.2f}"), arrow(ra.get("refres_r_meas"), rb.get("refres_r_meas"), lambda v: f"{100 * v:.1f}%"), arrow(ra.get("refres_lowres_r_meas"), rb.get("refres_lowres_r_meas"), lambda v: f"{100 * v:.1f}%"), arrow(ra.get("wall_s"), rb.get("wall_s"), lambda v: f"{v:.0f}"), ", ".join(ch)]) doc.table(head, out, row_class=lambda r: "fail" if "verdict" in r[-1] else "") def crystals(rs): """Rows that describe a crystal - the no-crystal controls would only pollute a distribution.""" return [r for r in rs if "control" not in r.get("tags", [])] def scored(rs): return [r for r in rs if r["verdict"] in ("pass", "fail")] def status(r): """The verdict, except that a pass carried by an accepted alternative reference (score.py, `alternatives`) is counted on its own, so a reader sees how many rows the battery passed on a second acceptable answer rather than on the deposition.""" return "accepted_alt" if r.get("cause") == "accepted_alternative" else r["verdict"] def rate(rs): s = scored(rs) n = sum(r["verdict"] == "pass" for r in s) return f"{n}/{len(s)} ({100 * n / len(s):.0f}%)" if s else "-" def build(run_dir, baseline=None, allow_incomplete=False): """Markdown and HTML report of one run, with the delta against a baseline run if given.""" man, res = load_run(run_dir, allow_incomplete) if baseline: check_baseline(baseline, man.get("private")) fill_depdata(run_dir, res) b = man["binary"] doc = Doc(f"Rugnux battery - {os.path.basename(os.path.normpath(run_dir))}") if man.get("private"): doc.p("PRIVATE ARM - confidential user data. This report stays in the run directory; it " "must not be copied into the repository, an issue or any public report.") state = run_state(man) if state != "complete": why = f" ({man['aborted']['reason']})" if man.get("aborted") else "" doc.p(f"RUN {state.upper()}{why}: {len(res)} of {expected_rows(man)} rows have a " "result. Not a reference and not comparable with a finished run.") if man.get("results_schema", 1) < 3: doc.p("A run from before the one-command battery: shown by its `bare` rows (with the `model` " "rows' R-factors on the open arm); its `xds` rows have no counterpart and are left out.") doc.ul([ f"rugnux {b.get('version', '?')}, sha256 {b['sha256'][:16]}, build flags: {b.get('flags') or 'unknown'}", f"runner {man['runner_git']}, host {man['host']}, {man['start']} -> {man.get('end') or 'NOT FINISHED'}", f"arms: {', '.join(man['arms'])}; {len(man['sets'])} sets" + (f" (subset: {man['subset']})" if man.get("subset") else " (full arm)"), "timing: NOT a reference - the GPU was shared" if man.get("gpu_shared") else "timing: GPU was not shared during the run", f"command options: {' '.join(man['options']['extra_args']) or '(none beyond output selection)'}", ] + [f"command, {arm} arm: {d}" for arm, d in man.get("command_doc", {}).items()]) arms = [arm for arm in man["arms"] if any(r["arm"] == arm for r in res)] order = {arm: i for i, arm in enumerate(arms)} doc.h(2, "Summary") head = ["arm", "sets", "pass", "of which alt", "fail", "unscored", "not run", "pass rate", "median res. gain", "median ISa", "median R_meas", "median time s", "total time min"] rows = [] for arm in arms: rs = [r for r in res if r["arm"] == arm] cnt = {x: sum(r["verdict"] == x for r in rs) for x in ("pass", "fail", "unscored", "not_run")} cnt["accepted_alt"] = sum(status(r) == "accepted_alt" for r in rs) xs = crystals(rs) rows.append([arm, len(rs), cnt["pass"], cnt["accepted_alt"], cnt["fail"], cnt["unscored"], cnt["not_run"], rate(rs), f(median(r.get("res_gain_pct") for r in xs), "{:+.1f}%"), f(median(r.get("isa") for r in xs), "{:.1f}"), pct(median(r.get("r_meas") for r in xs)), f(median(r.get("wall_s") for r in rs), "{:.0f}"), f(sum(r.get("wall_s") or 0 for r in rs) / 60, "{:.1f}")]) doc.table(head, rows, num=range(1, 13)) doc.svg(verdict_bars([(arm, {x: sum(status(r) == x and r["arm"] == arm for r in res) for x in ("pass", "accepted_alt", "fail", "unscored", "not_run")}) for arm in arms]), "verdicts per arm") doc.p("Pass rate is over the scored sets (pass + fail). `of which alt` counts the passes that " "are already in `pass`: rows where the answer matched an accepted alternative reference " "rather than the deposition, listed under Accepted alternatives below. In the bar chart " "they are drawn as their own segment. Every number in this table is rugnux's own " "run at its own resolution cut: what a user gets. Resolution gain is (reference d_min - our " "d_min) / reference d_min: positive = finer than the deposition, or than XDS. Medians leave " "out the no-crystal controls. R_meas and CC1/2 are pooled over each run's own resolution " "range, so they describe a run and do not rank two; the like-for-like comparison with XDS " "is the reference-range table below. Each set runs once, so every time includes reading " "its images from disk (unless they were still in the page cache from before the run).") doc.h(3, "By population") rows = [] for arm in arms: rs = [r for r in res if r["arm"] == arm] for tag in sorted({t for r in rs for t in r.get("tags", [])}): sub = [r for r in rs if tag in r.get("tags", [])] rows.append([arm, tag, len(sub), rate(sub), f(median(r.get("res_gain_pct") for r in sub), "{:+.1f}%")]) doc.table(["arm", "population", "sets", "pass rate", "median res. gain"], rows, num=(2, 4)) doc.h(3, "Distributions (sets that merged, controls left out)") rows = [] for arm in arms: rs = crystals(r for r in res if r["arm"] == arm) for name, lab, fm in (("isa", "ISa", "{:.1f}"), ("r_meas", "R_meas", None), ("cc_half", "CC1/2", "{:.3f}"), ("res_gain_pct", "res. gain %", "{:+.1f}"), ("isa_ratio", "ref-range ISa / XDS", "{:.3f}"), ("r_meas_ratio", "ref-range R_meas / XDS", "{:.3f}"), ("lowres_r_meas", "low-res shell R_meas", None), ("lowres_r_meas_ratio", "ref-range low-res R_meas / XDS", "{:.3f}"), ("cc_half_noise_ratio", "ref-range CC1/2 noise / XDS", "{:.2f}"), ("rmodel_shell_scaled", "R_model, shell-scaled", "{:.3f}"), ("rfree", "R_free (placement, within-run only)", "{:.3f}"), ("radial_misfit", "radial misfit", "{:.2f}"), ("rfree_ratio", "R_free ratio", "{:.3f}"), ("refmac_rfree_ratio", "REFMAC R_free ratio", "{:.3f}"), ("dep_cc_delta_all", "dCC vs depositor, all", "{:+.3f}"), ("dep_cc_delta_outer", "dCC vs depositor, outer 2 shells", "{:+.3f}"), ("dep_beyond_cc", "CC(Fc^2) past the deposited limit", "{:.3f}"), ("wall_s", "time s", "{:.0f}")): q = quartiles(r.get(name) for r in rs) if q: cells = [pct(x) if fm is None else f(x, fm) for x in q] rows.append([arm, lab, sum(r.get(name) is not None for r in rs)] + cells) doc.table(["arm", "metric", "n", "min", "q1", "median", "q3", "max"], rows, num=range(2, 8)) doc.h(3, "Per set, against the reference") for arm in arms: pts = [(r["set"], r["d_min"] / r["d_min_ref"], r["verdict"], f"d_min {r['d_min']:.2f} vs {r['d_min_ref']:.2f} A, {r['verdict']}") for r in res if r["arm"] == arm and r.get("d_min") and r.get("d_min_ref")] if pts: doc.svg(ratio_dots(pts, f"{arm}: d_min ratio"), f"{arm}: d_min(rugnux, own cut) / d_min(reference), one point per set, sorted; below 1 " "= finer than the reference") for name, what, cap in ( ("isa_ratio", "ISa", "ISa of the reference-range table (the error model refitted on it) / " "XDS's ISa; above 1 = rugnux's error model is the better one"), ("r_meas_ratio", "R_meas", "R_meas over the reference range / XDS's R_meas; below 1 = " "rugnux's merge is the more consistent one"), ("lowres_r_meas_ratio", "low-res R_meas", "R_meas of the lowest-resolution shell of the " "reference-range table / of XDS's table; below 1 = " "rugnux's strong reflections agree better"), ("cc_half_noise_ratio", "CC1/2 noise", "half-set noise over the reference range read off CC1/2 " "(1 / CC1/2 - 1) / XDS's; 2 = as noisy as XDS's merge " "would be with half its observations (not scored)")): for arm in arms: pts = [(r["set"], r[name], r["verdict"], f"{what} ratio, {refres_note(r)}") for r in res if r["arm"] == arm and r.get(name)] if pts: doc.svg(ratio_dots(pts, f"{arm}: reference-range {what} ratio"), f"{arm}: {cap}. One point per set, sorted.") pts = [(r["set"], r["rfree_ratio"], r["verdict"], f"R_free {r['rfree']:.3f} vs deposited {r['rfree_deposited']:.3f}") for r in res if r.get("rfree_ratio")] if pts: doc.svg(ratio_dots(pts, "R_free ratio"), "R_free of the deposited model against our merge, as rugnux reports it with --model (a " "rigid-body placement, scored on rugnux's own free set: a trend number, not a refinement) / " "the R_free the depositor published, one point per set, sorted; below 1 = lower than published") pts = [(r["set"], r["refmac_rfree_ratio"], r["verdict"], f"R_free {r['refmac_rfree_depflags']:.3f} vs {r['refmac_rfree_depdata']:.3f} on the deposited data") for r in res if r.get("refmac_rfree_ratio")] if pts: doc.svg(ratio_dots(pts, "REFMAC R_free ratio"), "REFMAC check (--model-check): R_free of the deposited model against our merge / against " "the deposited structure factors, both on the depositor's free set with the same protocol, " "one point per set, sorted; below 1 = our data fit the model better") dep = [r for r in res if r.get("dep_status") == "ok"] if dep: doc.h(3, "Against the depositor's data") doc.p("Per resolution shell, the rank correlation of our merged intensities with |Fc|^2 minus that of the depositor's own " "data, on the reflections both carry (d < 4 A, eight shells of equal count), |Fc| from " "the deposited model as it is (no bulk solvent, no refinement; depdata_check.py). The " "model was refined against the depositor's data, so zero is already a good result: " "across a corpus the median sits near -0.01. 'Past the limit' is CC(I, |Fc|^2) of our " "reflections in the outermost of up to three shells beyond the deposited data's limit, " "which the model never saw: clearly above zero there is signal. Reported, never scored.") rows = [] for r in sorted(dep, key=lambda r: r["dep_cc_delta_outer"]): rows.append([r["set"], r["dep_kind"], f(r.get("d_min")), f(r.get("dep_d_min")), r["dep_n_common"], f(r["dep_cc_delta_all"], "{:+.3f}"), f(r["dep_cc_delta_outer"], "{:+.3f}"), f(r.get("dep_beyond_cc"), "{:.3f}"), f(r.get("dep_beyond_d")), f(r.get("refmac_rfree_ratio"), "{:.3f}"), f(r.get("isa"), "{:.1f}"), ", ".join(r.get("tags", []))]) doc.table(["set", "dep. data", "our d_min", "dep. d_min", "common refl.", "dCC all", "dCC outer 2", "CC past limit", "to d A", "REFMAC ratio", "ISa", "tags"], rows, num=range(2, 11)) other = [r for r in res if r.get("dep_status") not in (None, "ok")] if other: doc.p("No comparison: " + "; ".join(f"{r['set']} ({r['dep_reason']})" for r in other)) xds = [r for r in res if r.get("refres_range")] if xds: doc.h(3, "Like for like with XDS: the reference-range table") doc.p("The same merge, binned a second time over XDS's range (--report-resolution; report-only, " "nothing was processed differently for it), against XDS's CORRECT.LP totals. Where " "rugnux's own cut is coarser than the reference ('coverage' in the last column), the " "shells past it are not merged at all: the completeness there is coverage of XDS's " "range, not a quality loss, and the other rugnux numbers are over the shells it reached. " "XDS's totals are over its own merged range, which is the reference range except where " "the reference d_min was derived (see below).") rows = [] for r in sorted(xds, key=lambda r: (order.get(r["arm"], 9), r["set"])): rows.append([r["set"], r["arm"], r["refres_range"], f(r.get("d_min")), f(r.get("refres_completeness"), "{:.1f}") + " / " + f(r.get("completeness_ref"), "{:.1f}"), f(r.get("refres_multiplicity"), "{:.1f}") + " / " + f(r.get("multiplicity_ref"), "{:.1f}"), f(r.get("refres_i_over_sigma"), "{:.1f}"), pct(r.get("refres_r_meas")) + " / " + pct(r.get("r_meas_ref")), lowres_cell(r), f(r.get("refres_cc_half"), "{:.4f}") + " / " + f(r.get("cc_half_ref"), "{:.3f}"), f(r.get("cc_half_noise_ratio"), "{:.2f}"), f(r.get("refres_isa"), "{:.1f}") + " / " + f(r.get("isa_ref"), "{:.1f}"), refres_note(r)]) doc.table(["set", "arm", "range A", "own d_min", "compl % (rugnux / XDS)", "mult", "I/sigma", "R_meas", "low-res R_meas (to d A)", "CC1/2", "CC1/2 noise ratio", "ISa", "reading"], rows, num=(3, 6, 10)) doc.p("Data quality is not scored; this table is for a human to judge. " "CC1/2 = S / (S + E) (signal and half-set error variances), so 1 / CC1/2 - 1 = E / S is the " "half-set noise; the noise ratio is rugnux's over XDS's, with XDS's CC1/2 taken at the " "bottom of its printed rounding (-0.0005); 2 would mean noisier than XDS's merge with half " "its observations. The low-resolution R_meas is the lowest shell of each table (the shells " "are XDS's, so both cover the same reflections); reported like everything here.") derived = {r["set"]: r for r in res if (r.get("d_min_ref_rule") or "xds_range") != "xds_range"} if derived: doc.p("XDS merged past its own signal (CC1/2 of its finest shell not significant), so the " "reference d_min is where XDS's CC1/2 falls through 0.30, and that is also the d_min of the " "reference-range table: " + "; ".join(f"{s} (XDS range {f(r.get('d_min_xds'))} A, reference " f"{f(r.get('d_min_ref'))} A)" for s, r in sorted(derived.items())) + ". XDS's pooled R_meas, CC1/2 and completeness are over its whole range.") doc.h(3, "Failures") fails = [r for r in res if r["verdict"] == "fail"] doc.table(["set", "arm", "cause", "reason"], [[r["set"], r["arm"], r["cause"], r["reason"]] for r in fails]) other = [r for r in res if r["verdict"] in ("unscored", "not_run")] if other: doc.p("Not scored: " + "; ".join(f"{r['set']} ({r['reason']})" for r in other)) alt = [r for r in res if status(r) == "accepted_alt"] if alt: doc.h(3, "Accepted alternatives") doc.p(f"{len(alt)} of the passes above are rows that accept more than one reference: the " "deposition and our reduction disagree, the disagreement is real, and no test " "available to us settles it, so either answer passes as long as the program picks " "one of them. These are open questions, not errors attributed to the deposition; " "the manifest's `ref` keeps the deposited values verbatim in every one of them.") doc.ul(f"{r['set']} ({r['arm']}): we report {r.get('sg') or '-'}, the reference is " f"{r.get('sg_ref') or '-'}; accepted as {r['accepted_alt']}. {r['accepted_alt_why']}" for r in sorted(alt, key=lambda r: (r["arm"], r["set"]))) doc.h(3, "Per set") doc.p("Space group: on the open arm the data's own determination. Where --model put the model's " "enantiomorph on the label, the label is in the note and the group scored is the search's.") if any(r.get("rmodel_shell_scaled") for r in res): doc.p("R_free and R_work describe one run against its model and do not carry across runs: the " "model is scaled to the data by an overall factor and an anisotropic B, so whatever the " "amplitudes' radial profile does that this shape cannot follow is reported as R, and a " "change in the profile alone moves R_free further than a real change in the data does. " "R_model (shell-scaled) is the same R with one free scale per resolution shell removing " "exactly that, over every reflection rather than the free 5%, and it is the column to " "read in a delta table. Radial misfit is how big that per-shell rescale had to be (the " "RMS of its logarithm): when it moves between two runs, R_free between them means " "little. Both are reported for a human to judge; nothing is scored on them.") model = [("R_model (shell-scaled)", "rmodel_shell_scaled"), ("radial misfit", "radial_misfit"), ("CC model", "cc_model"), ("R_free (within-run)", "rfree"), ("R_work", "rwork"), ("dep R_free", "rfree_deposited"), ("R_free ratio", "rfree_ratio")] if any(r.get("rfree") for r in res) else [] model += [("REFMAC R_free", "refmac_rfree_depflags"), ("REFMAC dep data", "refmac_rfree_depdata"), ("REFMAC ratio", "refmac_rfree_ratio")] if any(r.get("refmac_status") for r in res) else [] model += [("dCC dep. outer", "dep_cc_delta_outer")] if any(r.get("dep_status") for r in res) else [] head = ["set", "arm", "verdict", "space group", "ref", "cell dev %", "V ratio", "d_min", "ref d_min", "gain", "compl %", "mult", "R_meas", "low-res R_meas", "CC1/2", "ISa", "ref ISa", "idx"] + \ [h for h, _ in model] + (["model fit"] if model else []) + ["time s", "note"] rows = [] for r in sorted(res, key=lambda r: (order.get(r["arm"], 9), r["set"])): note = "; ".join(x for x in (r.get("reason"), r.get("sg_label") and f"labelled {r['sg_label']} from the model") if x) rows.append([r["set"], r["arm"], r["verdict"], r.get("sg") or "-", r.get("sg_ref") or "-", f(r.get("cell_dev_pct")), f(r.get("volume_ratio"), "{:.3f}"), f(r.get("d_min")), f(r.get("d_min_ref")), f(r.get("res_gain_pct"), "{:+.1f}%"), f(r.get("completeness"), "{:.1f}"), f(r.get("multiplicity"), "{:.1f}"), pct(r.get("r_meas")), pct(r.get("lowres_r_meas")), f(r.get("cc_half"), "{:.3f}"), f(r.get("isa"), "{:.1f}"), f(r.get("isa_ref"), "{:.1f}"), pct(r.get("indexing_rate"), 0)] + [f(r.get(k), "{:.3f}") for _, k in model] + ([r.get("model_fit") or "-"] if model else []) + [f(r.get("wall_s"), "{:.0f}"), note]) doc.table(head, rows, num=range(5, len(head) - 1), row_class=lambda row: "fail" if row[2] == "fail" else "", cell_class=lambda i, x: {"pass": "pass", "fail": "fail"}.get(x) if i == 2 else None) if baseline: bman, bres = load_run(baseline) doc.h(2, f"Delta vs baseline {os.path.basename(os.path.normpath(baseline))}") doc.p(f"Baseline rugnux {bman['binary'].get('version', '?')}. Pass rates on the sets both runs have:") both = {key(r) for r in res} & {key(r) for r in bres} rows = [] for arm in arms: ra = [r for r in bres if key(r) in both and r["arm"] == arm] rb = [r for r in res if key(r) in both and r["arm"] == arm] rows.append([arm, len(rb), rate(ra), rate(rb)]) doc.table(["arm", "common sets", "baseline", "this run"], rows, num=(1,)) doc.p("Rows whose result moved beyond noise (thresholds in report.py NOISE; confirm with " "`compare --rerun-changed` before believing a single-set change):") compare_table(doc, compare_rows(bres, res)) return doc def lowres_cell(r): """rugnux / XDS R_meas of the lowest-resolution shell, with the shells' high-resolution limits.""" return (pct(r.get("refres_lowres_r_meas")) + " / " + pct(r.get("lowres_r_meas_ref")) + f" ({f(r.get('refres_lowres_d'))} / {f(r.get('lowres_d_ref'))})") def refres_note(r): """How to read a row of the reference-range table.""" if r.get("refres_shells_past_limit"): return (f"coverage: own cut {f(r.get('d_min'))} A is coarser than the reference, " f"{r['refres_shells_past_limit']:.0f} shell(s) not merged") return "like for like"