leonarski_fandClaude Opus 5.5
d481da0474
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C
372 lines
20 KiB
Python
372 lines
20 KiB
Python
"""Scoring one rugnux run against its reference.
|
|
|
|
One dataset, one verdict, one cause, decided in this order: did it run -> is the lattice right ->
|
|
is the symmetry right. Data quality is not scored: it is reported for a human to judge. The order matters:
|
|
a halved axis also makes the screw along it unobservable, and counting that row twice would hide
|
|
the defect that cost the reflections.
|
|
|
|
A row may accept more than one reference (`ref_alternatives`, see `alternatives`): where two
|
|
descriptions of the same crystal are both defensible and nothing available to us decides between
|
|
them, either answer passes, and the report says which rows those are.
|
|
"""
|
|
import itertools
|
|
import os
|
|
import re
|
|
|
|
import gemmi
|
|
|
|
import sgequiv
|
|
|
|
REPORT_LINE = re.compile(r"^([A-Z_0-9]+)=\s*(.*)$")
|
|
|
|
|
|
def read_report(path):
|
|
"""KEY= value lines of a rugnux _report.txt, plus the lowest-resolution shell of its two shell
|
|
tables: LOWRES_D / LOWRES_R_MEAS from the table over its own range, REFRES_LOWRES_D /
|
|
REFRES_LOWRES_R_MEAS from the one over the reference range (it follows REFRES_RANGE=)."""
|
|
out = {}
|
|
if not os.path.exists(path):
|
|
return out
|
|
header = None
|
|
for line in open(path, errors="replace"):
|
|
line = line.rstrip("\n")
|
|
m = REPORT_LINE.match(line)
|
|
if m:
|
|
out[m.group(1)] = m.group(2).strip()
|
|
continue
|
|
f = line.lstrip("#").split()
|
|
if f[:2] == ["d_min", "N_obs"]:
|
|
header = f
|
|
elif header and f and re.match(r"^\d+\.\d+$", f[0]):
|
|
# Three tables in report order: the run's own, the --report-resolution one, and the
|
|
# per-hand (FRIEDEL'S_LAW=FALSE) one, whose keys are written just above it.
|
|
prefix = ("ANOM_" if "ANOM_UNIQUE_REFLECTIONS" in out
|
|
else "REFRES_" if "REFRES_RANGE" in out else "")
|
|
out[prefix + "LOWRES_D"] = f[0]
|
|
r_meas = f[header.index("R_meas")].rstrip("%") # "3.7%", or "-" for none
|
|
out[prefix + "LOWRES_R_MEAS"] = "" if r_meas == "-" else str(round(float(r_meas) / 100, 4))
|
|
header = None # only the first row of each table
|
|
return out
|
|
|
|
|
|
# REFRES_* keys of the report read as numbers, stored lower-case under the same name
|
|
REFRES_KEYS = ("refres_shells_past_limit", "refres_unique_reflections", "refres_completeness",
|
|
"refres_multiplicity", "refres_i_over_sigma", "refres_r_meas", "refres_cc_half",
|
|
"refres_isa", "refres_r_meas_weighted")
|
|
|
|
|
|
def fnum(s):
|
|
try:
|
|
return float(s.split()[0])
|
|
except (AttributeError, ValueError, IndexError):
|
|
return None
|
|
|
|
|
|
def space_group(name, sgno):
|
|
"""The space group in the SETTING the cell was given in: by name when there is one (I 1 2 1 and
|
|
C 1 2 1 are both number 5, with different centring vectors), else the standard setting."""
|
|
sg = gemmi.find_spacegroup_by_name(name) if name else None
|
|
return sg or gemmi.find_spacegroup_by_number(int(sgno or 1))
|
|
|
|
|
|
def primitive_reduced(cell, sg):
|
|
"""Niggli-reduced PRIMITIVE cell of a cell given in space group sg (a gemmi.SpaceGroup), and
|
|
its volume.
|
|
|
|
Setting-invariant: two settings of one lattice (C2 vs I2, a/c swapped, beta vs 180-beta)
|
|
reduce to the same cell, so they compare equal."""
|
|
gv = gemmi.GruberVector(gemmi.UnitCell(*cell), sg)
|
|
gv.niggli_reduce()
|
|
red = gv.cell_parameters()
|
|
return red, gemmi.UnitCell(*red).volume
|
|
|
|
|
|
def lattice_match(cell, sg, ref_cell, ref_sg):
|
|
"""(primitive volume ratio, largest relative deviation of the reduced edges)."""
|
|
red, vol = primitive_reduced(cell, sg)
|
|
ref_red, ref_vol = primitive_reduced(ref_cell, ref_sg)
|
|
edge_dev = max(abs(a - b) / b for a, b in zip(sorted(red[:3]), sorted(ref_red[:3])))
|
|
return vol / ref_vol, edge_dev
|
|
|
|
|
|
def cell_dev_pct(cell, ref):
|
|
"""Largest relative deviation of a, b, c in percent, minimised over the six axis orders, so
|
|
P212121 with a and c exchanged is not scored as a 6% error."""
|
|
return min(max(abs(p[i] - ref[i]) / ref[i] * 100 for i in range(3))
|
|
for p in itertools.permutations(cell[:3]))
|
|
|
|
|
|
# XDS prints CC1/2 in % with one decimal; its value is taken at the bottom of that rounding, so a
|
|
# printed 100.0 does not demand a perfect CC1/2 of rugnux
|
|
XDS_CC_HALF_ROUNDING = 0.0005
|
|
|
|
|
|
def cc_half_noise_ratio(cc, cc_xds):
|
|
"""rugnux's half-set noise over XDS's, both read off CC1/2 over XDS's range; None without both.
|
|
|
|
CC1/2 = S / (S + E), with S the variance of the signal and E that of the half-set error, so
|
|
E / S = 1 / CC1/2 - 1 at any CC1/2. E goes as 1 / (observations): a ratio of 2 is a merge as
|
|
noisy as XDS's would be with half of its observations. Reported, not scored.
|
|
A CC1/2 at or below 0 is no signal at all, counted as 0.001."""
|
|
if cc is None or cc_xds is None:
|
|
return None
|
|
return round((1 / max(cc, 0.001) - 1) / (1 / (cc_xds - XDS_CC_HALF_ROUNDING) - 1), 3)
|
|
|
|
|
|
_AXIS_ROW = {"a": (1, 0, 0), "b": (0, 1, 0), "c": (0, 0, 1)}
|
|
|
|
|
|
def screw_axes_differing(sg, ref_sg):
|
|
"""The axes on which two space groups disagree about a screw: the principal row one of them
|
|
extinguishes and the other does not.
|
|
|
|
Only ever asked of two groups of the SAME point group on the same lattice, so their centrings
|
|
are the same and a row a centring already kills is killed in both - what is left is the screw."""
|
|
out = set()
|
|
for ax, hkl in _AXIS_ROW.items():
|
|
a, b = (s.operations().is_systematically_absent(hkl) for s in (sg, ref_sg))
|
|
if a != b:
|
|
out.add(ax)
|
|
return out
|
|
|
|
|
|
def point_group(sgno):
|
|
return gemmi.find_spacegroup_by_number(int(sgno)).point_group_hm()
|
|
|
|
|
|
def sg_name(sgno):
|
|
return gemmi.find_spacegroup_by_number(int(sgno)).hm if sgno else None
|
|
|
|
|
|
def alternatives(entry):
|
|
"""The other answers a row accepts, from its `ref_alternatives`.
|
|
|
|
A knife-edge row - one where the deposition and our reduction disagree, the disagreement is
|
|
real, and no test available to us settles it - says so here instead of being quietly overridden
|
|
to our answer or dropped from scoring. Each alternative replaces the reference fields it names
|
|
(a space group, a cell, or both) and the row passes if our answer matches ANY of the
|
|
references, the deposited one included. `ref` keeps the deposition verbatim either way.
|
|
|
|
Every alternative must carry `why`: a reason a reader can check. Without one this would be a
|
|
way to launder a failure into a pass, so it is a schema error, not a silent pass."""
|
|
out = []
|
|
for i, alt in enumerate(entry.get("ref_alternatives") or []):
|
|
where = f"{entry.get('id', '?')}: ref_alternatives[{i}]"
|
|
if not (alt.get("why") or "").strip():
|
|
raise ValueError(f"{where} has no 'why'; an accepted alternative must state why both "
|
|
"answers are acceptable")
|
|
if not set(alt) & {"sg", "sgno", "cell"}:
|
|
raise ValueError(f"{where} replaces nothing; give the sg, the cell, or both")
|
|
out.append(alt)
|
|
return out
|
|
|
|
|
|
def alt_label(alt):
|
|
"""How an accepted alternative is named in one line: its space group, its cell, or both."""
|
|
bits = []
|
|
if alt.get("sg") or alt.get("sgno"):
|
|
bits.append(alt.get("sg") or sg_name(alt["sgno"]))
|
|
if alt.get("cell"):
|
|
bits.append("cell " + " ".join(f"{x:g}" for x in alt["cell"][:3]))
|
|
return ", ".join(bits)
|
|
|
|
|
|
def judge(entry, rep, run_note):
|
|
"""Score one set. entry is its manifest row (with 'ref'), rep its parsed report, run_note
|
|
what the runner saw ('', 'timeout', 'exit N: message', 'no input')."""
|
|
ref = dict(entry.get("ref") or {})
|
|
if entry.get("ref_override"): # a reference measured by hand, replacing the file's
|
|
ref.update(entry["ref_override"])
|
|
alts = alternatives(entry) # raises on a row that accepts an answer for no reason
|
|
r = {
|
|
"sgno": None, "sg": None, "pg": None,
|
|
"sgno_ref": ref.get("sgno"), "sg_ref": ref.get("sg") or sg_name(ref.get("sgno")),
|
|
"pg_ref": point_group(ref["sgno"]) if ref.get("sgno") else None,
|
|
"cell": None, "cell_ref": ref.get("cell"), "cell_dev_pct": None, "volume_ratio": None,
|
|
"d_min": None, "d_min_ref": ref.get("dmin"), "d_min_ref_rule": ref.get("dmin_rule"),
|
|
"d_min_xds": ref.get("dmin_xds"), "res_gain_pct": None,
|
|
"r_meas": fnum(rep.get("R_MEAS")), "cc_half": fnum(rep.get("CC_HALF")),
|
|
# R_MEAS is counted as XDS counts it; the weighted one weights each observation as the
|
|
# merge does (absent from reports before the weighting)
|
|
"r_meas_weighted": fnum(rep.get("R_MEAS_WEIGHTED")),
|
|
"isa": fnum(rep.get("ISA")), "completeness": fnum(rep.get("COMPLETENESS")),
|
|
"mosaicity_deg": fnum(rep.get("MOSAICITY_DEG")), "extra_lattice_intensity_pct": fnum(rep.get("EXTRA_LATTICE_INTENSITY_PCT")),
|
|
"multiplicity": fnum(rep.get("MULTIPLICITY")), "i_over_sigma": fnum(rep.get("I_OVER_SIGMA")),
|
|
"indexing_rate": fnum(rep.get("INDEXING_RATE")), "images": fnum(rep.get("IMAGES_PROCESSED")),
|
|
"rugnux_wall_s": fnum(rep.get("WALL_TIME")), "rugnux_verdict": rep.get("VERDICT_TEXT"),
|
|
"r_meas_ref": ref.get("r_meas"), "cc_half_ref": ref.get("cc_half"), "isa_ref": ref.get("isa"),
|
|
"completeness_ref": ref.get("completeness"), "multiplicity_ref": ref.get("multiplicity"),
|
|
"sg_relation": None,
|
|
# rugnux's own fit of the deposited model (--model, open arm): a rigid-body placement scored
|
|
# on rugnux's own free set, so a trend number, not the depositor's R-free
|
|
"rfree": fnum(rep.get("R_FREE")), "rwork": fnum(rep.get("R_WORK")),
|
|
"model_fit": rep.get("MODEL_FIT"), "cc_model": fnum(rep.get("CC_MODEL_OVERALL")),
|
|
# R over ALL the reflections, with and without one free scale per resolution shell, and the
|
|
# size of that rescale. rfree above describes one run and is not comparable with another's:
|
|
# the model's scale can only bend as k*exp(-B s^2), so a change in the amplitudes' radial
|
|
# profile alone moves it. rmodel_shell_scaled has that taken out and IS comparable; it is
|
|
# the R to read in a delta table. radial_misfit says how much there was to take out.
|
|
"rmodel": fnum(rep.get("R_MODEL")), "rmodel_shell_scaled": fnum(rep.get("R_MODEL_SHELL_SCALED")),
|
|
"radial_misfit": fnum(rep.get("MODEL_RADIAL_MISFIT")),
|
|
"sg_label": None, "accepted_alt": None, "accepted_alt_why": None,
|
|
}
|
|
# the same merge binned over the reference's range (--report-resolution, XDS arms)
|
|
r.update({k: fnum(rep.get(k.upper())) for k in REFRES_KEYS})
|
|
r["refres_range"] = rep.get("REFRES_RANGE")
|
|
# An XDS reference measured with FRIEDEL'S_LAW=FALSE counts its uniques, completeness,
|
|
# multiplicity, R_meas and CC1/2 per Bijvoet hand, so what is like for like is the per-hand
|
|
# table of the same rugnux merge - the Friedel-merged one has about twice the multiplicity.
|
|
# Both tables come out of one bare run: -A is presentation only and changes nothing that is
|
|
# compared here. ISa is not among them - there is one error model, fitted on the hands, and
|
|
# REFRES_ISA is it.
|
|
r["per_hand_reference"] = False
|
|
if ref.get("anomalous"):
|
|
for k in ("unique_reflections", "completeness", "multiplicity", "i_over_sigma",
|
|
"r_meas", "cc_half"):
|
|
v = fnum(rep.get("ANOM_REFRES_" + k.upper()))
|
|
if v is not None:
|
|
r["refres_" + k] = v
|
|
r["per_hand_reference"] = True
|
|
r["isa_ratio"] = round(r["refres_isa"] / r["isa_ref"], 4) if r["refres_isa"] and r["isa_ref"] else None
|
|
r["r_meas_ratio"] = (round(r["refres_r_meas"] / r["r_meas_ref"], 4)
|
|
if r["refres_r_meas"] and r["r_meas_ref"] else None)
|
|
# R_meas of the lowest-resolution shell: rugnux's own table, the reference-range table and XDS's
|
|
# (reported, not scored)
|
|
r.update(lowres_d=fnum(rep.get("LOWRES_D")), lowres_r_meas=fnum(rep.get("LOWRES_R_MEAS")),
|
|
refres_lowres_d=fnum(rep.get("REFRES_LOWRES_D")),
|
|
refres_lowres_r_meas=fnum(rep.get("REFRES_LOWRES_R_MEAS")),
|
|
lowres_d_ref=ref.get("dmin_low"), lowres_r_meas_ref=ref.get("r_meas_low"))
|
|
r["lowres_r_meas_ratio"] = (round(r["refres_lowres_r_meas"] / r["lowres_r_meas_ref"], 4)
|
|
if r["refres_lowres_r_meas"] and r["lowres_r_meas_ref"] else None)
|
|
r["cc_half_noise_ratio"] = cc_half_noise_ratio(r["refres_cc_half"], r["cc_half_ref"])
|
|
if rep.get("SPACE_GROUP_NUMBER"):
|
|
r["sgno"] = int(fnum(rep["SPACE_GROUP_NUMBER"]))
|
|
r["sg"] = rep.get("SPACE_GROUP_NAME") or sg_name(r["sgno"])
|
|
# With --model the reported group can be the model's enantiomorph, a label the data did not
|
|
# decide. The data's own answer is then the search's Sohncke group (same class up to hand).
|
|
if rep.get("SPACE_GROUP_ENANTIOMORPH") == "ASSUMED_FROM_MODEL" and rep.get("SOHNCKE_SPACE_GROUP"):
|
|
own = gemmi.find_spacegroup_by_name(rep["SOHNCKE_SPACE_GROUP"])
|
|
if own and own.number != r["sgno"] and sgequiv.indistinguishable(r["sgno"], own.number):
|
|
r["sg_label"] = r["sg"]
|
|
r["sgno"], r["sg"] = own.number, own.hm
|
|
r["pg"] = point_group(r["sgno"])
|
|
if rep.get("UNIT_CELL_CONSTANTS"):
|
|
r["cell"] = [float(x) for x in rep["UNIT_CELL_CONSTANTS"].split()[:6]]
|
|
rng = (rep.get("INCLUDE_RESOLUTION_RANGE") or "").split()
|
|
if len(rng) == 2:
|
|
r["d_min"] = float(rng[1])
|
|
if r["d_min"] and r["d_min_ref"]:
|
|
r["res_gain_pct"] = round((r["d_min_ref"] - r["d_min"]) / r["d_min_ref"] * 100, 1) + 0.0 # no -0.0
|
|
if r["cell"] and r["cell_ref"]:
|
|
r["cell_dev_pct"] = round(cell_dev_pct(r["cell"], r["cell_ref"]), 3)
|
|
|
|
def verdict(v, cause, reason):
|
|
r.update(verdict=v, cause=cause, reason=reason)
|
|
return r
|
|
|
|
have_lattice = r["sgno"] is not None and r["cell"] is not None
|
|
if entry.get("expect") == "no_lattice":
|
|
# a no-crystal control: the right answer is to refuse
|
|
if have_lattice:
|
|
return verdict("fail", "false_lattice",
|
|
f"reported a lattice ({r['sg']}) on a set with no crystal")
|
|
return verdict("pass", None, "no lattice reported, as expected")
|
|
if not have_lattice:
|
|
if run_note == "no input":
|
|
return verdict("not_run", "no_input", "input file not found")
|
|
if run_note == "timeout":
|
|
return verdict("fail", "timeout", "timed out")
|
|
if re.search(r"Error reading input|Cannot open", run_note):
|
|
return verdict("fail", "reader", run_note)
|
|
if re.search(r"index|lattice", run_note, re.I):
|
|
return verdict("fail", "indexing", run_note)
|
|
return verdict("fail", "crash", run_note or "no lattice in the report")
|
|
if not ref.get("sgno") or not ref.get("cell"):
|
|
return verdict("unscored", "no_reference", "no reference to score against")
|
|
if entry.get("unscored"): # a reference known to be wrong: run, but do not score
|
|
return verdict("unscored", "reference_problem", entry["unscored"])
|
|
|
|
def against(cand):
|
|
"""Score the answer against one reference: (the numbers it yields, None) if that reference
|
|
accepts the answer, (the numbers, (verdict, cause, reason)) if it does not."""
|
|
n = {"volume_ratio": None, "sg_relation": None}
|
|
vr, edge = lattice_match(r["cell"], space_group(rep.get("SPACE_GROUP_NAME"), r["sgno"]),
|
|
cand["cell"], space_group(cand.get("sg"), cand["sgno"]))
|
|
n["volume_ratio"] = round(vr, 4)
|
|
if not (0.95 < vr < 1.05 and edge < 0.02):
|
|
if 0.45 <= vr <= 0.55:
|
|
return n, ("fail", "lattice_halved", f"primitive volume ratio {vr:.2f}")
|
|
if 1.9 <= vr <= 2.1:
|
|
return n, ("fail", "lattice_doubled", f"primitive volume ratio {vr:.2f}")
|
|
return n, ("fail", "lattice_other",
|
|
f"primitive volume ratio {vr:.2f}, reduced edges off by {edge:.1%}")
|
|
|
|
# XDS never tests a screw axis, so against an XDS reference only the point group can be
|
|
# judged - unless the group was measured by hand (sg_measured).
|
|
by_sg = entry["arm"] == "open" or cand.get("sg_measured")
|
|
pg_cand = point_group(cand["sgno"])
|
|
if by_sg:
|
|
rel = sgequiv.describe(cand["sgno"], r["sgno"])
|
|
n["sg_relation"] = rel
|
|
ok = rel in ("match", "hand only (needs anomalous)", "UNDECIDABLE from intensities")
|
|
else:
|
|
ok = r["pg"] == pg_cand
|
|
if ok:
|
|
return n, None
|
|
sg_cand = cand.get("sg") or sg_name(cand["sgno"])
|
|
order = len(gemmi.find_spacegroup_by_number(r["sgno"]).operations().sym_ops)
|
|
order_ref = len(gemmi.find_spacegroup_by_number(cand["sgno"]).operations().sym_ops)
|
|
if r["pg"] == pg_cand:
|
|
# A screw the data never tested is not a wrong answer, it is an unanswerable question:
|
|
# the axial row it lives on was not recorded at all (the spindle's blind cone), so the
|
|
# run offers every member of the set as an alternative and writes the one claiming no
|
|
# screw, because a reflection file must carry one group. Scoring that as a failure
|
|
# measures the sweep, not the program - the same reason the hand of an enantiomorphic
|
|
# pair is not scored. Three conditions, all necessary: rugnux SAYS the axis is open,
|
|
# the reference is among the groups it offered, and the ONLY axis the two differ on is
|
|
# one it named. A screw called wrongly where the row WAS measured meets none of them
|
|
# and stays a failure.
|
|
# WHICH axis carries the disagreement is a property of the setting, and the two sides
|
|
# are named in different ones: the reference in the deposition's axis labels, ours - and
|
|
# SPACE_GROUP_SCREW_UNDETERMINED with it - in our own. Where the cells are a permutation
|
|
# of each other the two never meet: a deposited P 21 21 2 differs from our P 21 21 21 on
|
|
# c, while the same group written in our axes differs on b, the axis actually left open.
|
|
# So the axes are compared against the offered group, which is already in our setting;
|
|
# the reference being among the offered is a condition anyway, and a group offered under
|
|
# more than one setting passes if any of them fits.
|
|
undetermined = set((rep.get("SPACE_GROUP_SCREW_UNDETERMINED") or "NONE").split())
|
|
offered = [gemmi.find_spacegroup_by_name(x.strip())
|
|
for x in (rep.get("SPACE_GROUP_ALTERNATIVES") or "").split("|")]
|
|
ours = space_group(rep.get("SPACE_GROUP_NAME"), r["sgno"])
|
|
for o in (x for x in offered if x and x.number == cand["sgno"]):
|
|
differ = screw_axes_differing(ours, o)
|
|
if undetermined != {"NONE"} and differ and differ <= undetermined:
|
|
here = f" (here {o.hm})" if o.hm != sg_cand else ""
|
|
return n, ("unscored", "screw_undetermined",
|
|
f"{r['sg']} vs reference {sg_cand}{here}: the screw along "
|
|
f"{', '.join(sorted(differ))} is undeterminable from these data "
|
|
f"(offered {rep.get('SPACE_GROUP_ALTERNATIVES')})")
|
|
return n, ("fail", "sym_screw", f"{r['sg']} vs reference {sg_cand}")
|
|
cause = ("sym_under" if order < order_ref else
|
|
"sym_over" if order > order_ref else "sym_other")
|
|
return n, ("fail", cause, f"{r['sg']} vs reference {sg_cand}")
|
|
|
|
# The deposition (with any ref_override) decides, and its numbers are the ones reported: an
|
|
# accepted alternative changes the verdict, never what the row says about the deposition.
|
|
numbers, problem = against(ref)
|
|
r.update(numbers)
|
|
for alt in (alts if problem and problem[0] == "fail" else []):
|
|
if against(dict(ref, **{k: v for k, v in alt.items() if k != "why"}))[1] is None:
|
|
r["accepted_alt"], r["accepted_alt_why"] = alt_label(alt), alt["why"]
|
|
return verdict("pass", "accepted_alternative",
|
|
f"{r['sg']} vs reference {r['sg_ref']}: accepted alternative "
|
|
f"{r['accepted_alt']} - a knife-edge this battery does not decide")
|
|
if problem:
|
|
return verdict(*problem)
|
|
|
|
# Data quality (CC1/2, R_meas, resolution, ISa) is reported, never scored: that is for a human.
|
|
note = ""
|
|
if r["sgno"] != ref["sgno"]:
|
|
note = f"{r['sg']} vs reference {r['sg_ref']} ({r.get('sg_relation') or 'screws not judged against XDS'})"
|
|
return verdict("pass", None, note)
|