Jungfraujoch/tools/battery/report.py

"""The per-run report and the run-to-run comparison."""
import json
import multiprocessing
import os
import socket
import statistics
import sys

import gemmi

import depdata_check
import nullmodel
import score
import shelx_check
import timing
from render import Doc, ratio_dots, verdict_bars

# A change smaller than these is treated as run-to-run noise. They are a starting point, to be
# re-calibrated from `compare --rerun-changed`, which measures the noise directly.
NOISE = {
    "d_min": ("rel", 0.02),         # 2% of the resolution limit
    "isa": ("rel", 0.05),
    "r_meas": ("rel", 0.05),
    "cc_half": ("abs", 0.005),
    "completeness": ("abs", 1.0),   # percentage points
    "cell_dev_pct": ("abs", 0.2),   # percentage points
    "rfree": ("abs", 0.005),
    "rmodel_shell_scaled": ("abs", 0.005),
    "radial_misfit": ("abs", 0.05),
    "anom_sigma": ("rel", 0.05),
    "refres_r_meas": ("rel", 0.05),
    "refres_isa": ("rel", 0.05),
    "refres_lowres_r_meas": ("rel", 0.05),
    "wall_s": ("rel", 0.25),        # and at least 10 s - see beyond_noise()
    "shx_r1": ("abs", 0.003),       # SHELXL refinement of a small molecule's COD structure
    "shx_wr2": ("abs", 0.005),
    "shx_goof": ("abs", 0.03),
    "shx_exti": ("abs", 0.02),
}


def run_state(man):
    """complete, aborted (marked so), running (its runner is alive), or unfinished (killed
    without a trace - treat as aborted)."""
    if man.get("complete"):
        return "complete"
    if man.get("aborted"):
        return "aborted"
    # (not on Windows, where os.kill does not probe: it terminates the process, or sends it Ctrl-C)
    if man.get("pid") and man.get("host") == socket.gethostname() and os.name != "nt":
        try:
            os.kill(man["pid"], 0)
            return "running"
        except OSError:
            pass
    return "unfinished"


def load_run(run_dir, allow_incomplete=False):
    """A run's manifest and results. A run that did not finish covers an arbitrary part of its
    sets and is refused unless asked for."""
    man = json.load(open(os.path.join(run_dir, "manifest.json")))
    state = run_state(man)
    if state != "complete" and not allow_incomplete:
        sys.exit(f"{run_dir} is {state}, not a finished run; pass --allow-incomplete to use it anyway")
    path = os.path.join(run_dir, "results.json")
    res = json.load(open(path)) if os.path.exists(path) else []
    schema = man.get("results_schema", 1)
    if schema < 2:
        schema1(man, res)
    if schema < 3:
        res[:] = schema2(res)
    else:
        rescore(run_dir, man, res)
    return man, res


def rescore(run_dir, man, res):
    """Score every row again from its saved report, with today's scorer and today's manifest rows
    (references, `unscored`, `expect`), so a report or a compare never mixes two scorings or two
    references. results.json keeps the verdicts as they were scored when the run finished.

    A row whose report has no lattice keeps its verdict: why it failed (reader, timeout) came from
    the run, not from the report. A crash is read again from the run's log (score.crash_note), so a
    run scored before the crash verdict existed shows its crashes as such too."""
    own = {(e["arm"], e["id"]): e for e in man["sets"]}
    for r in res:
        wd = os.path.join(run_dir, "work", r["arm"], r["set"])
        crash = score.crash_note(r.get("exit_code"), wd)
        if crash:
            r.update(verdict="crash", cause="crash", reason=crash)
            continue
        e = MANIFESTS.get(r["arm"], {}).get(key(r)[1]) or own.get((r["arm"], r["set"]))
        rep = score.read_report(os.path.join(wd, "p_report.txt"))
        lattice = rep.get("SPACE_GROUP_NUMBER") and rep.get("UNIT_CELL_CONSTANTS")
        if e and (lattice or e.get("expect") == "no_lattice"):
            r.update(score.judge(dict(e, arm=r["arm"]), rep, ""))


def _depdata(job):
    os.nice(10)
    return depdata_check.check_set(*job)


def fill_depdata(run_dir, res):
    """The depositor-data comparison (depdata_check.py) for open-arm rows of a run made before it
    existed, computed from the saved merge, four sets at a time."""
    todo = [r for r in res if r["arm"] == "open" and "dep_status" not in r and r.get("model")
            and os.path.exists(os.path.join(run_dir, "work", "open", r["set"], "p.mtz"))]
    if not todo:
        return
    print(f"depositor-data comparison for {len(todo)} sets of an older run ...", file=sys.stderr)
    jobs = [(os.path.join(run_dir, "work", "open", r["set"]), r["set"],
             PDB_CACHE or depdata_check.model_check.DEFAULT_CACHE) for r in todo]
    with multiprocessing.Pool(4) as pool:
        for r, out in zip(todo, pool.map(_depdata, jobs, chunksize=1)):
            r.update(out)


def _shelx(job):
    os.nice(10)
    return shelx_check.check_tmp(*job)


def fill_shelx(run_dir, man, res):
    """The SHELXL refinement (shelx_check.py) for small-molecule rows of a run made before it
    existed, from the saved p.hkl, four sets at a time. The COD id comes from today's manifest."""
    own = {(e["arm"], e["id"]): e for e in man["sets"]}
    todo = []
    for r in res:
        e = MANIFESTS.get(r["arm"], {}).get(key(r)[1]) or own.get((r["arm"], r["set"])) or {}
        wd = os.path.join(run_dir, "work", r["arm"], r["set"])
        if e.get("cod") and "shx_status" not in r and os.path.exists(os.path.join(wd, "p.hkl")):
            todo.append((r, (wd, e["cod"], COD_CACHE or shelx_check.DEFAULT_CACHE, SHELXL)))
    if not todo or not SHELXL:
        return
    print(f"SHELXL refinement for {len(todo)} small-molecule sets of an older run ...", file=sys.stderr)
    with multiprocessing.Pool(4) as pool:
        for (r, _), out in zip(todo, pool.map(_shelx, [j for _, j in todo], chunksize=1)):
            r.update(out)


def schema1(man, res):
    """Read a run from before the variants (results schema 1) as one: it ran each set once, plain
    on the open arm and with XDS's settings on the XDS arms - unless --unforced, which left the
    range free but still passed -A, and is read as `bare`. Its R-factors came from the REFMAC check."""
    for r in res:
        r.setdefault("variant", "xds" if r["arm"] != "open" and man["options"].get("xds_range") else "bare")
        if "model_status" in r:
            for k in ("rfree", "rwork", "rfree_depflags", "rfree_depdata", "rfree_ratio"):
                r["refmac_" + k] = r.pop(k, None)
            r["refmac_status"], r["refmac_reason"] = r.pop("model_status"), r.pop("model_reason", None)
            r["rfree_deposited"] = None


def schema2(res):
    """Read a run with variants (schema 2) as one run per set: its `bare` row, with the open arm's
    `model` row's R-factors folded in. That is today's processing on the open arm; on the XDS arms
    today's command adds -A where XDS was anomalous. The `xds` rows (XDS's range forced on the
    processing) have no counterpart and are left out."""
    model = {r["set"]: r for r in res if r["variant"] == "model"}
    out = []
    for r in res:
        if r["variant"] != "bare":
            continue
        m = model.get(r["set"])
        if m:
            r.update({k: m.get(k) for k in ("rfree", "rwork", "model_fit", "rfree_deposited", "rfree_ratio")})
        out.append(r)
    return out


def expected_rows(man):
    """Rows a finished run has: one per set (per set and variant before schema 3)."""
    if man.get("results_schema", 1) >= 3:
        return len(man["sets"])
    return sum(len(man.get("variants", {}).get(e["arm"], [None])) for e in man["sets"])


def check_baseline(baseline, private):
    """A baseline must have finished, and be as private as the run it is compared with: the delta
    table names its sets."""
    man, _ = load_run(baseline)
    if bool(man.get("private")) != bool(private):
        sys.exit(f"baseline {baseline} is {'' if man.get('private') else 'not '}a private run; "
                 "a report may only be compared with a run of its own kind")


# Version of the results.json row schema (README.md, "results.json"). Bump it when a field changes
# meaning or is removed; adding a field does not need it.
# 2: one row per set AND variant (`variant`, `first_read`); `rfree`/`rwork`/`rfree_ratio` are rugnux's
#    own --model R-factors, the REFMAC check's moved to `refmac_*`.
# 3: one row per set again (one command per arm, no `variant`/`first_read`); `refres_*` added;
#    `sgno`/`sg` are the data's own group where --model relabelled the hand (`sg_label`).
RESULTS_SCHEMA = 3

# {arm: {old set id: current id}} from the manifests (battery.py sets it), so a run made before a
# dataset was renamed still pairs with one made after
ALIASES = {}

# {arm: {set id: manifest row}} of today's manifests (battery.py sets it); rescore() scores with them
MANIFESTS = {}

# the site's cache of deposited models and structure factors (battery.py sets it)
PDB_CACHE = None

# the site's cache of COD structures and the SHELXL binary (battery.py sets them; None: no SHELXL)
COD_CACHE = None
SHELXL = None


def key(r):
    return (r["arm"], ALIASES.get(r["arm"], {}).get(r["set"], r["set"]))


def f(x, fmt="{:.2f}", dash="-"):
    return dash if x is None else fmt.format(x)


def pct(x, nd=1):
    return "-" if x is None else f"{100 * x:.{nd}f}%"


def median(xs):
    xs = [x for x in xs if x is not None]
    return statistics.median(xs) if xs else None


def quartiles(xs):
    xs = sorted(x for x in xs if x is not None)
    if not xs:
        return None
    if len(xs) < 4:
        return xs[0], xs[len(xs) // 2], xs[len(xs) // 2], xs[len(xs) // 2], xs[-1]
    q = statistics.quantiles(xs, n=4)
    return xs[0], q[0], q[1], q[2], xs[-1]


def beyond_noise(name, a, b):
    if a is None or b is None:
        return (a is None) != (b is None)
    kind, tol = NOISE[name]
    d = abs(b - a)
    if name == "wall_s" and d < 10:
        return False
    return d > (tol * abs(a) if kind == "rel" else tol)


def changes(ra, rb):
    """What changed between two results of the same set, beyond noise."""
    out = []
    if ra["verdict"] != rb["verdict"]:
        out.append(f"verdict {ra['verdict']}->{rb['verdict']}")
    if ra.get("sgno") != rb.get("sgno"):
        out.append("space group")
    if ra.get("volume_ratio") and rb.get("volume_ratio") and \
            abs(ra["volume_ratio"] - rb["volume_ratio"]) > 0.05:
        out.append("lattice")
    for name in ("d_min", "isa", "r_meas", "cc_half", "completeness", "cell_dev_pct"):
        if beyond_noise(name, ra.get(name), rb.get(name)):
            out.append(name)
    # The R that is comparable between runs, and the radial profile that decides whether `rfree`
    # beside it can be read at all (both only where both runs carry them).
    for name in ("rmodel_shell_scaled", "radial_misfit", "rfree", "anom_sigma"):
        if ra.get(name) is not None and rb.get(name) is not None and beyond_noise(name, ra[name], rb[name]):
            out.append(name)
    # the like-for-like table only where both runs have one (a pre-schema-3 run has none)
    for name in ("refres_r_meas", "refres_isa", "refres_lowres_r_meas"):
        if ra.get(name) is not None and rb.get(name) is not None and beyond_noise(name, ra[name], rb[name]):
            out.append(name)
    for name in ("shx_r1", "shx_wr2", "shx_goof", "shx_exti"):
        if ra.get(name) is not None and rb.get(name) is not None and beyond_noise(name, ra[name], rb[name]):
            out.append("SHELXL " + name[4:])
    if beyond_noise("wall_s", ra.get("wall_s"), rb.get("wall_s")):
        out.append("time")
    return out


def compare_rows(res_a, res_b):
    """[(key, ra, rb, changes)] over the sets both runs have, then those only one has."""
    a = {key(r): r for r in res_a}
    b = {key(r): r for r in res_b}
    rows = [(k, a[k], b[k], changes(a[k], b[k])) for k in sorted(a) if k in b]
    rows += [(k, a[k], None, ["only in A"]) for k in sorted(a) if k not in b]
    rows += [(k, None, b[k], ["only in B"]) for k in sorted(b) if k not in a]
    return rows


def arrow(x, y, fmt=lambda v: str(v)):
    x = "-" if x is None else fmt(x)
    y = "-" if y is None else fmt(y)
    return x if x == y else f"{x} -> {y}"


def compare_table(doc, rows, only_changed=True):
    head = ["set", "arm", "verdict", "space group", "d_min", "ISa", "R_meas", "CC1/2",
            "cell dev %", "R_model (shell-scaled)", "R_free", "radial misfit", "anom. sigma",
            "ref-range R_meas", "ref-range low-res R_meas", "SHELXL R1", "SHELXL wR2", "SHELXL GooF",
            "SHELXL EXTI", "time s", "beyond noise"]
    out = []
    for (arm, s), ra, rb, ch in rows:
        if only_changed and not ch:
            continue
        ra, rb = ra or {}, rb or {}
        out.append([s, arm, arrow(ra.get("verdict"), rb.get("verdict")),
                    arrow(ra.get("sg"), rb.get("sg")),
                    arrow(ra.get("d_min"), rb.get("d_min"), lambda v: f"{v:.2f}"),
                    arrow(ra.get("isa"), rb.get("isa"), lambda v: f"{v:.1f}"),
                    arrow(ra.get("r_meas"), rb.get("r_meas"), lambda v: f"{100 * v:.1f}%"),
                    arrow(ra.get("cc_half"), rb.get("cc_half"), lambda v: f"{v:.3f}"),
                    arrow(ra.get("cell_dev_pct"), rb.get("cell_dev_pct"), lambda v: f"{v:.2f}"),
                    arrow(ra.get("rmodel_shell_scaled"), rb.get("rmodel_shell_scaled"), lambda v: f"{v:.3f}"),
                    arrow(ra.get("rfree"), rb.get("rfree"), lambda v: f"{v:.3f}"),
                    arrow(ra.get("radial_misfit"), rb.get("radial_misfit"), lambda v: f"{v:.2f}"),
                    arrow(ra.get("anom_sigma"), rb.get("anom_sigma"), lambda v: f"{v:.2f}"),
                    arrow(ra.get("refres_r_meas"), rb.get("refres_r_meas"), lambda v: f"{100 * v:.1f}%"),
                    arrow(ra.get("refres_lowres_r_meas"), rb.get("refres_lowres_r_meas"),
                          lambda v: f"{100 * v:.1f}%"),
                    arrow(ra.get("shx_r1"), rb.get("shx_r1"), lambda v: f"{v:.4f}"),
                    arrow(ra.get("shx_wr2"), rb.get("shx_wr2"), lambda v: f"{v:.4f}"),
                    arrow(ra.get("shx_goof"), rb.get("shx_goof"), lambda v: f"{v:.3f}"),
                    arrow(ra.get("shx_exti"), rb.get("shx_exti"), lambda v: f"{v:.3f}"),
                    arrow(ra.get("wall_s"), rb.get("wall_s"), lambda v: f"{v:.0f}"),
                    ", ".join(ch)])
    doc.table(head, out, row_class=lambda r: "fail" if "verdict" in r[-1] else "")


def crystals(rs):
    """Rows that describe a crystal - the no-crystal controls would only pollute a distribution."""
    return [r for r in rs if "control" not in r.get("tags", [])]


def scored(rs):
    """Rows that count towards a pass rate: a crash counts as a failure."""
    return [r for r in rs if r["verdict"] in ("pass", "fail", "crash")]


def status(r):
    """The verdict, except that a pass carried by an accepted alternative reference (score.py,
    `alternatives`) is counted on its own, so a reader sees how many rows the battery passed on a
    second acceptable answer rather than on the deposition."""
    return "accepted_alt" if r.get("cause") == "accepted_alternative" else r["verdict"]


def hardware_line(man):
    """One line for the report header; old runs have no "hardware" key."""
    hw = man.get("hardware")
    if not hw:
        return "hardware: not recorded"
    freq_bits = [f"{hw['cpu_base_ghz']:.2f} GHz nominal ({hw['cpu_base_source']})" if hw.get("cpu_base_ghz")
                 else "nominal frequency unknown"]
    if hw.get("cpu_max_boost_ghz"):
        freq_bits.append(f"{hw['cpu_max_boost_ghz']:.2f} GHz max boost")
    freq = ", ".join(freq_bits)
    cores = f"{f(hw.get('physical_cores'), '{}')}C/{hw.get('threads') or '?'}T, {f(hw.get('sockets'), '{}')} socket(s)"
    mem = f"{hw['memory_gib']:.0f} GiB RAM" if hw.get("memory_gib") else "RAM unknown"
    gpus = ", ".join(f"{g['name']} ({g['memory']})" for g in hw.get("gpus") or []) or "none"
    return f"hardware: {hw.get('cpu_model', 'unknown')}, {freq}, {cores}, {mem}, GPU: {gpus}"


def rate(rs):
    s = scored(rs)
    n = sum(r["verdict"] == "pass" for r in s)
    return f"{n}/{len(s)} ({100 * n / len(s):.0f}%)" if s else "-"


def build(run_dir, baseline=None, allow_incomplete=False):
    """Markdown and HTML report of one run, with the delta against a baseline run if given."""
    man, res = load_run(run_dir, allow_incomplete)
    if baseline:
        check_baseline(baseline, man.get("private"))
    fill_depdata(run_dir, res)
    fill_shelx(run_dir, man, res)
    b = man["binary"]
    doc = Doc(f"Rugnux battery - {os.path.basename(os.path.normpath(run_dir))}")
    if man.get("private"):
        doc.p("PRIVATE ARM - confidential user data. This report stays in the run directory; it "
              "must not be copied into the repository, an issue or any public report.")
    state = run_state(man)
    if state != "complete":
        why = f" ({man['aborted']['reason']})" if man.get("aborted") else ""
        doc.p(f"RUN {state.upper()}{why}: {len(res)} of {expected_rows(man)} rows have a "
              "result. Not a reference and not comparable with a finished run.")
    if man.get("results_schema", 1) < 3:
        doc.p("A run from before the one-command battery: shown by its `bare` rows (with the `model` "
              "rows' R-factors on the open arm); its `xds` rows have no counterpart and are left out.")
    doc.ul([
        f"rugnux {b.get('version', '?')}, sha256 {b['sha256'][:16]}, build flags: {b.get('flags') or 'unknown'}",
        f"runner {man['runner_git']}, host {man['host']}, {man['start']} -> {man.get('end') or 'NOT FINISHED'}",
        hardware_line(man),
        f"arms: {', '.join(man['arms'])}; {len(man['sets'])} sets"
        + (f" (subset: {man['subset']})" if man.get("subset") else " (full arm)"),
        "timing: NOT a reference - the GPU was shared" if man.get("gpu_shared")
        else "timing: GPU was not shared during the run",
        f"command options: {' '.join(man['options']['extra_args']) or '(none beyond output selection)'}",
    ] + [f"command, {arm} arm: {d}" for arm, d in man.get("command_doc", {}).items()])

    arms = [arm for arm in man["arms"] if any(r["arm"] == arm for r in res)]
    order = {arm: i for i, arm in enumerate(arms)}

    crashed = [r for r in res if r["verdict"] == "crash"]
    if crashed:
        doc.h(2, f"CRASHED: {len(crashed)} set(s)")
        doc.p("A crash fails the run whatever else it did, and counts as a failure in every pass rate. "
              "One that does not reproduce is a race to find, not noise to rerun away "
              "(`battery.py stress`).")
        doc.table(["set", "arm", "why"], [[r["set"], r["arm"], r["reason"]] for r in crashed],
                  row_class=lambda row: "fail")

    null_path = os.path.join(run_dir, "nullmodel_results.json")
    if os.path.exists(null_path):
        nullmodel.section(doc, json.load(open(null_path)))
    timing.section(doc, res)
    if not res:   # the wrong-model null on its own (battery.py nullmodel): that section is the report
        return doc

    doc.h(2, "Summary")
    head = ["arm", "sets", "pass", "of which alt", "fail", "crash", "unscored", "not run", "pass rate",
            "median res. gain", "median ISa", "median R_meas", "median time s", "total time min"]
    rows = []
    for arm in arms:
        rs = [r for r in res if r["arm"] == arm]
        cnt = {x: sum(r["verdict"] == x for r in rs) for x in ("pass", "fail", "crash", "unscored", "not_run")}
        cnt["accepted_alt"] = sum(status(r) == "accepted_alt" for r in rs)
        xs = crystals(rs)
        rows.append([arm, len(rs), cnt["pass"], cnt["accepted_alt"], cnt["fail"], cnt["crash"], cnt["unscored"],
                     cnt["not_run"], rate(rs),
                     f(median(r.get("res_gain_pct") for r in xs), "{:+.1f}%"),
                     f(median(r.get("isa") for r in xs), "{:.1f}"),
                     pct(median(r.get("r_meas") for r in xs)),
                     f(median(r.get("wall_s") for r in rs), "{:.0f}"),
                     f(sum(r.get("wall_s") or 0 for r in rs) / 60, "{:.1f}")])
    doc.table(head, rows, num=range(1, 14))
    doc.svg(verdict_bars([(arm, {x: sum(status(r) == x and r["arm"] == arm for r in res)
                                 for x in ("pass", "accepted_alt", "fail", "crash", "unscored", "not_run")})
                          for arm in arms]), "verdicts per arm")
    doc.p("Pass rate is over the scored sets (pass + fail + crash). `of which alt` counts the passes that "
          "are already in `pass`: rows where the answer matched an accepted alternative reference "
          "rather than the deposition, listed under Accepted alternatives below. In the bar chart "
          "they are drawn as their own segment. Every number in this table is rugnux's own "
          "run at its own resolution cut: what a user gets. Resolution gain is (reference d_min - our "
          "d_min) / reference d_min: positive = finer than the deposition, or than XDS. Medians leave "
          "out the no-crystal controls. R_meas and CC1/2 are pooled over each run's own resolution "
          "range, so they describe a run and do not rank two; the like-for-like comparison with XDS "
          "is the reference-range table below. Each set runs once, so every time includes reading "
          "its images from disk (unless they were still in the page cache from before the run).")

    doc.h(3, "By population")
    rows = []
    for arm in arms:
        rs = [r for r in res if r["arm"] == arm]
        for tag in sorted({t for r in rs for t in r.get("tags", [])}):
            sub = [r for r in rs if tag in r.get("tags", [])]
            rows.append([arm, tag, len(sub), rate(sub), f(median(r.get("res_gain_pct") for r in sub), "{:+.1f}%")])
    doc.table(["arm", "population", "sets", "pass rate", "median res. gain"], rows, num=(2, 4))

    doc.h(3, "Distributions (sets that merged, controls left out)")
    rows = []
    for arm in arms:
        rs = crystals(r for r in res if r["arm"] == arm)
        for name, lab, fm in (("isa", "ISa", "{:.1f}"), ("r_meas", "R_meas", None),
                              ("cc_half", "CC1/2", "{:.3f}"), ("res_gain_pct", "res. gain %", "{:+.1f}"),
                              ("isa_ratio", "ref-range ISa / XDS", "{:.3f}"),
                              ("r_meas_ratio", "ref-range R_meas / XDS", "{:.3f}"),
                              ("lowres_r_meas", "low-res shell R_meas", None),
                              ("lowres_r_meas_ratio", "ref-range low-res R_meas / XDS", "{:.3f}"),
                              ("cc_half_noise_ratio", "ref-range CC1/2 noise / XDS", "{:.2f}"),
                              ("rmodel_shell_scaled", "R_model, shell-scaled", "{:.3f}"),
                              ("rfree", "R_free (placement, within-run only)", "{:.3f}"),
                              ("radial_misfit", "radial misfit", "{:.2f}"),
                              ("rfree_ratio", "R_free ratio", "{:.3f}"),
                              ("anom_sigma", "anomalous map at the scatterers, sigma", "{:.2f}"),
                              ("refmac_rfree_ratio", "REFMAC R_free ratio", "{:.3f}"),
                              ("dep_cc_delta_all", "dCC vs depositor, all", "{:+.3f}"),
                              ("dep_cc_delta_outer", "dCC vs depositor, outer 2 shells", "{:+.3f}"),
                              ("dep_beyond_cc", "CC(Fc^2) past the deposited limit", "{:.3f}"),
                              ("wall_s", "time s", "{:.0f}")):
            q = quartiles(r.get(name) for r in rs)
            if q:
                cells = [pct(x) if fm is None else f(x, fm) for x in q]
                rows.append([arm, lab, sum(r.get(name) is not None for r in rs)] + cells)
    doc.table(["arm", "metric", "n", "min", "q1", "median", "q3", "max"], rows, num=range(2, 8))

    doc.h(3, "Per set, against the reference")
    for arm in arms:
        pts = [(r["set"], r["d_min"] / r["d_min_ref"], r["verdict"],
                f"d_min {r['d_min']:.2f} vs {r['d_min_ref']:.2f} A, {r['verdict']}")
               for r in res if r["arm"] == arm and r.get("d_min") and r.get("d_min_ref")]
        if pts:
            doc.svg(ratio_dots(pts, f"{arm}: d_min ratio"),
                    f"{arm}: d_min(rugnux, own cut) / d_min(reference), one point per set, sorted; below 1 "
                    "= finer than the reference")
    for name, what, cap in (
            ("isa_ratio", "ISa", "ISa of the reference-range table (the error model refitted on it) / "
                                 "XDS's ISa; above 1 = rugnux's error model is the better one"),
            ("r_meas_ratio", "R_meas", "R_meas over the reference range / XDS's R_meas; below 1 = "
                                       "rugnux's merge is the more consistent one"),
            ("lowres_r_meas_ratio", "low-res R_meas", "R_meas of the lowest-resolution shell of the "
                                                      "reference-range table / of XDS's table; below 1 = "
                                                      "rugnux's strong reflections agree better"),
            ("cc_half_noise_ratio", "CC1/2 noise", "half-set noise over the reference range read off CC1/2 "
                                                   "(1 / CC1/2 - 1) / XDS's; 2 = as noisy as XDS's merge "
                                                   "would be with half its observations (not scored)")):
        for arm in arms:
            pts = [(r["set"], r[name], r["verdict"], f"{what} ratio, {refres_note(r)}")
                   for r in res if r["arm"] == arm and r.get(name)]
            if pts:
                doc.svg(ratio_dots(pts, f"{arm}: reference-range {what} ratio"),
                        f"{arm}: {cap}. One point per set, sorted.")
    pts = [(r["set"], r["rfree_ratio"], r["verdict"],
            f"R_free {r['rfree']:.3f} vs deposited {r['rfree_deposited']:.3f}")
           for r in res if r.get("rfree_ratio")]
    if pts:
        doc.svg(ratio_dots(pts, "R_free ratio"),
                "R_free of the deposited model against our merge, as rugnux reports it with --model (a "
                "rigid-body placement, scored on rugnux's own free set: a trend number, not a refinement) / "
                "the R_free the depositor published, one point per set, sorted; below 1 = lower than published")
    pts = [(r["set"], r["refmac_rfree_ratio"], r["verdict"],
            f"R_free {r['refmac_rfree_depflags']:.3f} vs {r['refmac_rfree_depdata']:.3f} on the deposited data")
           for r in res if r.get("refmac_rfree_ratio")]
    if pts:
        doc.svg(ratio_dots(pts, "REFMAC R_free ratio"),
                "REFMAC check (--model-check): R_free of the deposited model against our merge / against "
                "the deposited structure factors, both on the depositor's free set with the same protocol, "
                "one point per set, sorted; below 1 = our data fit the model better")

    dep = [r for r in res if r.get("dep_status") == "ok"]
    if dep:
        doc.h(3, "Against the depositor's data")
        doc.p("Per resolution shell, the rank correlation of our merged intensities with |Fc|^2 minus that of the depositor's own "
              "data, on the reflections both carry (d < 4 A, eight shells of equal count), |Fc| from "
              "the deposited model as it is (no bulk solvent, no refinement; depdata_check.py). The "
              "model was refined against the depositor's data, so zero is already a good result: "
              "across a corpus the median sits near -0.01. 'Past the limit' is CC(I, |Fc|^2) of our "
              "reflections in the outermost of up to three shells beyond the deposited data's limit, "
              "which the model never saw: clearly above zero there is signal. Reported, never scored.")
        rows = []
        for r in sorted(dep, key=lambda r: r["dep_cc_delta_outer"]):
            rows.append([r["set"], r["dep_kind"], f(r.get("d_min")), f(r.get("dep_d_min")),
                         r["dep_n_common"], f(r["dep_cc_delta_all"], "{:+.3f}"),
                         f(r["dep_cc_delta_outer"], "{:+.3f}"), f(r.get("dep_beyond_cc"), "{:.3f}"),
                         f(r.get("dep_beyond_d")), f(r.get("refmac_rfree_ratio"), "{:.3f}"),
                         f(r.get("isa"), "{:.1f}"), ", ".join(r.get("tags", []))])
        doc.table(["set", "dep. data", "our d_min", "dep. d_min", "common refl.", "dCC all",
                   "dCC outer 2", "CC past limit", "to d A", "REFMAC ratio", "ISa", "tags"],
                  rows, num=range(2, 11))
        other = [r for r in res if r.get("dep_status") not in (None, "ok")]
        if other:
            doc.p("No comparison: " + "; ".join(f"{r['set']} ({r['dep_reason']})" for r in other))

    xds = [r for r in res if r.get("refres_range")]
    if xds:
        doc.h(3, "Like for like with XDS: the reference-range table")
        doc.p("The same merge, binned a second time over XDS's range (--report-resolution; report-only, "
              "nothing was processed differently for it), against XDS's CORRECT.LP totals. Where "
              "rugnux's own cut is coarser than the reference ('coverage' in the last column), the "
              "shells past it are not merged at all: the completeness there is coverage of XDS's "
              "range, not a quality loss, and the other rugnux numbers are over the shells it reached. "
              "XDS's totals are over its own merged range, which is the reference range except where "
              "the reference d_min was derived (see below).")
        rows = []
        for r in sorted(xds, key=lambda r: (order.get(r["arm"], 9), r["set"])):
            rows.append([r["set"], r["arm"], r["refres_range"], f(r.get("d_min")),
                         f(r.get("refres_completeness"), "{:.1f}") + " / " + f(r.get("completeness_ref"), "{:.1f}"),
                         f(r.get("refres_multiplicity"), "{:.1f}") + " / " + f(r.get("multiplicity_ref"), "{:.1f}"),
                         f(r.get("refres_i_over_sigma"), "{:.1f}"),
                         pct(r.get("refres_r_meas")) + " / " + pct(r.get("r_meas_ref")),
                         lowres_cell(r),
                         f(r.get("refres_cc_half"), "{:.4f}") + " / " + f(r.get("cc_half_ref"), "{:.3f}"),
                         f(r.get("cc_half_noise_ratio"), "{:.2f}"),
                         f(r.get("refres_isa"), "{:.1f}") + " / " + f(r.get("isa_ref"), "{:.1f}"),
                         refres_note(r)])
        doc.table(["set", "arm", "range A", "own d_min", "compl % (rugnux / XDS)", "mult", "I/sigma",
                   "R_meas", "low-res R_meas (to d A)", "CC1/2", "CC1/2 noise ratio", "ISa", "reading"],
                  rows, num=(3, 6, 10))
        doc.p("Data quality is not scored; this table is for a human to judge. "
              "CC1/2 = S / (S + E) (signal and half-set error variances), so 1 / CC1/2 - 1 = E / S is the "
              "half-set noise; the noise ratio is rugnux's over XDS's, with XDS's CC1/2 taken at the "
              "bottom of its printed rounding (-0.0005); 2 would mean noisier than XDS's merge with half "
              "its observations. The low-resolution R_meas is the lowest shell of each table (the shells "
              "are XDS's, so both cover the same reflections); reported like everything here.")

    shx = [r for r in res if r.get("shx_status")]
    if shx:
        doc.h(3, "Small molecules: SHELXL refinement of the published structure")
        doc.p("The set's structure from the Crystallography Open Database (COD id) refined against our "
              "p.hkl by SHELXL with one fixed recipe (shelx_check.py: data reindexed into the COD "
              "setting, non-H anisotropic, H fixed at the COD positions, EXTI refined, three rounds with "
              "the weights SHELXL suggests, MERG 2), the way a chemical crystallographer judges data. "
              "WGHT a near its 0.2 ceiling and a large EXTI mean the strong reflections or the sigmas are "
              "off; K top is <Fo2>/<Fc2> of SHELXL's strongest analysis-of-variance bin (below 1: strong "
              "reflections read low). Fixed-model R1(F) scores the data against |Fc| of the COD model as "
              "published, one scale, nothing refined. R(int) means something only for unmerged data. "
              "Reported, never scored.")
        rows = []
        for r in sorted(shx, key=lambda r: (order.get(r["arm"], 9), r["set"])):
            rows.append([r["set"], r["arm"], r.get("shx_cod") or "-", f(r.get("d_min")),
                         f(r.get("shx_r1"), "{:.4f}") + (f" ({r['shx_n4']})" if r.get("shx_n4") else ""),
                         f(r.get("shx_wr2"), "{:.4f}"), f(r.get("shx_goof"), "{:.3f}"),
                         f(r.get("shx_exti"), "{:.3f}"),
                         f(r.get("shx_wght_a"), "{:.3f}") + " / " + f(r.get("shx_wght_b"), "{:.2f}"),
                         f(r.get("shx_peak")) + " / " + f(r.get("shx_hole")),
                         f(r.get("shx_rint"), "{:.4f}") + " / " + f(r.get("shx_rsigma"), "{:.4f}"),
                         f(r.get("shx_k_top"), "{:.3f}"), f(r.get("shx_fixed_r1"), "{:.4f}"),
                         r["shx_status"] if r["shx_status"] == "ok" else f"{r['shx_status']}: {r.get('shx_reason')}"])
        doc.table(["set", "arm", "COD", "own d_min", "R1 >4sig (n)", "wR2", "GooF", "EXTI", "WGHT a / b",
                   "peak / hole e/A3", "R(int) / R(sigma)", "K top", "fixed-model R1(F)", "status"],
                  rows, num=range(3, 13))

    derived = {r["set"]: r for r in res if (r.get("d_min_ref_rule") or "xds_range") != "xds_range"}
    if derived:
        doc.p("XDS merged past its own signal (CC1/2 of its finest shell not significant), so the "
              "reference d_min is where XDS's CC1/2 falls through 0.30, and that is also the d_min of the "
              "reference-range table: "
              + "; ".join(f"{s} (XDS range {f(r.get('d_min_xds'))} A, reference "
                          f"{f(r.get('d_min_ref'))} A)" for s, r in sorted(derived.items()))
              + ". XDS's pooled R_meas, CC1/2 and completeness are over its whole range.")

    doc.h(3, "Failures")
    fails = [r for r in res if r["verdict"] in ("fail", "crash")]
    doc.table(["set", "arm", "cause", "reason"],
              [[r["set"], r["arm"], r["cause"], r["reason"]] for r in fails])
    other = [r for r in res if r["verdict"] in ("unscored", "not_run")]
    if other:
        doc.p("Not scored: " + "; ".join(f"{r['set']} ({r['reason']})" for r in other))

    alt = [r for r in res if status(r) == "accepted_alt"]
    if alt:
        doc.h(3, "Accepted alternatives")
        doc.p(f"{len(alt)} of the passes above are rows that accept more than one reference: the "
              "deposition and our reduction disagree, the disagreement is real, and no test "
              "available to us settles it, so either answer passes as long as the program picks "
              "one of them. These are open questions, not errors attributed to the deposition; "
              "the manifest's `ref` keeps the deposited values verbatim in every one of them.")
        doc.ul(f"{r['set']} ({r['arm']}): we report {r.get('sg') or '-'}, the reference is "
               f"{r.get('sg_ref') or '-'}; accepted as {r['accepted_alt']}. {r['accepted_alt_why']}"
               for r in sorted(alt, key=lambda r: (r["arm"], r["set"])))

    doc.h(3, "Per set")
    doc.p("Space group: on the open arm the data's own determination. Where --model put the model's "
          "enantiomorph on the label, the label is in the note and the group scored is the search's.")
    if any(r.get("rmodel_shell_scaled") for r in res):
        doc.p("R_free and R_work describe one run against its model and do not carry across runs: the "
              "model is scaled to the data by an overall factor and an anisotropic B, so whatever the "
              "amplitudes' radial profile does that this shape cannot follow is reported as R, and a "
              "change in the profile alone moves R_free further than a real change in the data does. "
              "R_model (shell-scaled) is the same R with one free scale per resolution shell removing "
              "exactly that, over every reflection rather than the free 5%, and it is the column to "
              "read in a delta table. Radial misfit is how big that per-shell rescale had to be (the "
              "RMS of its logarithm): when it moves between two runs, R_free between them means "
              "little. Both are reported for a human to judge; nothing is scored on them.")
    if any(r.get("anom_sigma") for r in res):
        doc.p("Anom. sigma is the anomalous difference map (F+ - F- on the model's phases) read at the "
              "model's anomalous scatterers - every atom from phosphorus up: the S of Met and Cys, "
              "metals, Cl, I - and averaged, in units of the map's r.m.s.: how much anomalous signal "
              "the merge carries, on the model's yardstick. Reported, never scored.")
    model = [("R_model (shell-scaled)", "rmodel_shell_scaled"), ("radial misfit", "radial_misfit"),
             ("CC model", "cc_model"), ("R_free (within-run)", "rfree"), ("R_work", "rwork"),
             ("dep R_free", "rfree_deposited"),
             ("R_free ratio", "rfree_ratio"),
             ("anom. sigma", "anom_sigma")] if any(r.get("rfree") for r in res) else []
    model += [("REFMAC R_free", "refmac_rfree_depflags"), ("REFMAC dep data", "refmac_rfree_depdata"),
              ("REFMAC ratio", "refmac_rfree_ratio")] if any(r.get("refmac_status") for r in res) else []
    model += [("dCC dep. outer", "dep_cc_delta_outer")] if any(r.get("dep_status") for r in res) else []
    head = ["set", "arm", "verdict", "space group", "ref", "cell dev %", "V ratio", "d_min", "ref d_min",
            "gain", "compl %", "mult", "R_meas", "low-res R_meas", "CC1/2", "ISa", "ref ISa", "idx"] + \
           [h for h, _ in model] + (["model fit"] if model else []) + ["time s", "note"]
    rows = []
    for r in sorted(res, key=lambda r: (order.get(r["arm"], 9), r["set"])):
        note = "; ".join(x for x in (r.get("reason"), r.get("sg_label") and f"labelled {r['sg_label']} from the model")
                         if x)
        rows.append([r["set"], r["arm"], r["verdict"], r.get("sg") or "-", r.get("sg_ref") or "-",
                     f(r.get("cell_dev_pct")), f(r.get("volume_ratio"), "{:.3f}"),
                     f(r.get("d_min")), f(r.get("d_min_ref")), f(r.get("res_gain_pct"), "{:+.1f}%"),
                     f(r.get("completeness"), "{:.1f}"), f(r.get("multiplicity"), "{:.1f}"),
                     pct(r.get("r_meas")), pct(r.get("lowres_r_meas")), f(r.get("cc_half"), "{:.3f}"),
                     f(r.get("isa"), "{:.1f}"),
                     f(r.get("isa_ref"), "{:.1f}"), pct(r.get("indexing_rate"), 0)]
                    + [f(r.get(k), "{:.3f}") for _, k in model]
                    + ([r.get("model_fit") or "-"] if model else [])
                    + [f(r.get("wall_s"), "{:.0f}"), note])
    doc.table(head, rows, num=range(5, len(head) - 1),
              row_class=lambda row: "fail" if row[2] in ("fail", "crash") else "",
              cell_class=lambda i, x: {"pass": "pass", "fail": "fail", "crash": "fail"}.get(x) if i == 2 else None)

    if baseline:
        bman, bres = load_run(baseline)
        fill_shelx(baseline, bman, bres)
        doc.h(2, f"Delta vs baseline {os.path.basename(os.path.normpath(baseline))}")
        doc.p(f"Baseline rugnux {bman['binary'].get('version', '?')}. Pass rates on the sets both runs have:")
        both = {key(r) for r in res} & {key(r) for r in bres}
        rows = []
        for arm in arms:
            ra = [r for r in bres if key(r) in both and r["arm"] == arm]
            rb = [r for r in res if key(r) in both and r["arm"] == arm]
            rows.append([arm, len(rb), rate(ra), rate(rb)])
        doc.table(["arm", "common sets", "baseline", "this run"], rows, num=(1,))
        doc.p("Rows whose result moved beyond noise (thresholds in report.py NOISE; confirm with "
              "`compare --rerun-changed` before believing a single-set change):")
        compare_table(doc, compare_rows(bres, res))
    return doc


def lowres_cell(r):
    """rugnux / XDS R_meas of the lowest-resolution shell, with the shells' high-resolution limits."""
    return (pct(r.get("refres_lowres_r_meas")) + " / " + pct(r.get("lowres_r_meas_ref"))
            + f" ({f(r.get('refres_lowres_d'))} / {f(r.get('lowres_d_ref'))})")


def refres_note(r):
    """How to read a row of the reference-range table."""
    if r.get("refres_shells_past_limit"):
        return (f"coverage: own cut {f(r.get('d_min'))} A is coarser than the reference, "
                f"{r['refres_shells_past_limit']:.0f} shell(s) not merged")
    return "like for like"


def console_table(res):
    """One line per set for a terminal or a CI log, each pair rugnux / reference: the space group,
    then a tick where the set passed by score.judge's rule and a cross where it failed (a lattice,
    symmetry, crash or missing input; a crash is also listed under the table), d_min, ISa, CC1/2 overall and in the highest shell, R_meas
    of the lowest shell, the model's R-free (rugnux's placement / the model's published) and
    rugnux's own wall time. The references are XDS's on the XDS arms; the open arm has the
    deposition's space group, d_min and R-free only. Every pair is rugnux right-aligned, "/", the
    reference (right-aligned where it is a number), so the slashes, decimal points and ticks of a
    column line up."""
    def sg(no):
        return gemmi.find_spacegroup_by_number(no).short_name() if no else None

    def num(x, fmt, scale):
        return None if x is None else fmt.format(x * scale)

    # (header, key, reference key, format, scale, width of each side)
    pairs = (("space group", "sgno", "sgno_ref", None, 1, 7),
             ("d_min", "d_min", "d_min_ref", "{:.2f}", 1, 5),
             ("ISa", "isa", "isa_ref", "{:.1f}", 1, 5),
             ("CC1/2%", "cc_half", "cc_half_ref", "{:.1f}", 100, 5),
             ("hi CC1/2%", "cc_half_high", "cc_half_high_ref", "{:.1f}", 100, 5),
             ("lo Rmeas%", "lowres_r_meas", "lowres_r_meas_ref", "{:.1f}", 100, 5),
             ("R_free%", "rfree", "rfree_deposited", "{:.1f}", 100, 5))
    mark = {"pass": "✓", "fail": "✗", "crash": "✗", "not_run": "✗"}

    def line(name, arm, cells, tick, time):
        out = f"{name:18} {arm:7}"
        for n, (_, _, _, _, _, w) in enumerate(pairs):
            out += " " + cells[n]
            if n == 0:
                out += " " + tick
        return f"{out} {time:>6}"

    head = line("set", "arm", [h.center(2 * w + 1) for h, *_, w in pairs], " ", "time_s")
    out = [head.rstrip(), "-" * len(head)]
    for r in res:
        name = r["set"] if len(r["set"]) <= 18 else r["set"][:17] + "~"
        cells = []
        for _, k, kr, fmt, scale, w in pairs:
            a, b = ((sg(r.get(k)), sg(r.get(kr))) if fmt is None
                    else (num(r.get(k), fmt, scale), num(r.get(kr), fmt, scale)))
            # numbers right-aligned on both sides, so the decimal points line up too
            cells.append(f"{a or '-':>{w}}/" + (f"{b or '-':<{w}}" if fmt is None else f"{b or '-':>{w}}"))
        out.append(line(name, r["arm"], cells, mark.get(r["verdict"], "?"), f(r.get("wall_s"), "{:.0f}")))
    out += [f"CRASHED  {r['set']} ({r['arm']}): {r['reason']}" for r in res if r["verdict"] == "crash"]
    return "\n".join(out)