tools/battery: committed, reproducible validation battery
One runner for the open (vs PDB deposition) and in-house (vs XDS) arms, replacing the scattered scripts outside the repository. Each run writes a new, read-only directory named <date>_<rugnux git>_<label> holding a copy of the binary with its sha256 and build flags, manifest.json (inputs, references, options, host, GPU sharing) and results.json in one schema for all arms, plus per-set work directories and a Markdown and a self-contained HTML report (per-arm and per-population pass rates, resolution gain, ISa/R_meas distributions, timing, failures, delta vs a baseline run). - Dataset lists and references are data files: open.json (151 public sets, deposited group/cell/resolution, pinned inputs) and inhouse.json (standard test crystals and no-crystal controls, XDS references). The private arm's manifest stays outside the repository; a private run gets its own run directory and report. - Scoring is setting-invariant: lattice by Niggli-reduced primitive cell (beta vs 180-beta, C2 vs I2), symmetry via sgequiv on the open arm and by point group against XDS, then merge usability. Re-scoring the rc169 and rc170 battery outputs reproduces their verdicts. - Input discovery (`discover`) ranks sweeps by frame count, never by file size or sort order; a run never searches, it runs what the manifest names. - `compare A B` flags per-set changes beyond noise; `--rerun-changed` reruns them with A's saved binary to separate real changes from run-to-run noise. - `--tier smoke` selects a 22-set subset spanning the populations; a full-arm run takes a lock; `--gpulock` queues on the shared GPU and marks the timing as not a reference. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
leonarski_fandClaude Opus 5 committed
1 parent
30b6800289
commit
5e7b53dd7b
10 files changed
+1374
No files matched your search
@@ -39,3 +39,6 @@ common/GitInfo.cpp
|
||||
# Review reports, work plans and investigation notes. They are working material for one task, they
|
||||
# go stale the moment the code moves, and Sphinx would publish anything left under docs/.
|
||||
docs/review/
|
||||
|
||||
# Python bytecode (tools/battery and the root scripts)
|
||||
__pycache__/
|
||||
@@ -0,0 +1,445 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rugnux validation battery: run, score, report, compare.
|
||||
|
||||
Arms and where their truth comes from:
|
||||
open public PDB depositions (and a few published small-molecule sets); scored against the
|
||||
deposited space group, cell and resolution. Manifest: open.json (committed).
|
||||
inhouse standard test crystals measured at PSI (lysozyme, thaumatin, insulin, cytochrome C,
|
||||
myoglobin) plus no-crystal controls; scored against XDS (point group only - XDS never
|
||||
tests a screw axis). Manifest: inhouse.json (committed).
|
||||
private user data, scored like inhouse. Its manifest lives OUTSIDE the repository (path in the
|
||||
site config) and a private run gets its own run directory and report.
|
||||
|
||||
Usage:
|
||||
battery.py run --rugnux BIN --label L [--arm open,inhouse|private] [--only a,b | --tier smoke]
|
||||
[--build-dir DIR] [--gpulock] [--threads N] [--unforced] [--baseline RUN]
|
||||
battery.py compare RUN_A RUN_B [--all] [--rerun-changed]
|
||||
battery.py report RUN --out DIR [--baseline RUN] re-render a run's report
|
||||
battery.py discover [--refresh] open arm: propose inputs for new directories
|
||||
battery.py refs --arm inhouse|private [--write] re-read the XDS references from CORRECT.LP
|
||||
|
||||
A run writes <runs_root>/<YYYYMMDD-HHMM>_<rugnux git>_<label>/ holding a copy of the binary,
|
||||
manifest.json (what was run, on what, with which references), results.json (one row per set),
|
||||
report.md + report.html and work/<arm>/<set>/ (log, report, cif). It is made read-only when done.
|
||||
A full-arm run (no --only / --tier) takes the site lock and refuses to start if another holds it.
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
|
||||
import inputs # noqa: E402
|
||||
import report # noqa: E402
|
||||
import score # noqa: E402
|
||||
|
||||
DEFAULT_SITE = os.path.join(HERE, "site.json")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ configuration and manifests
|
||||
|
||||
def load_site(path):
|
||||
site = json.load(open(path))
|
||||
for arm in site["arms"].values():
|
||||
arm["manifest"] = os.path.join(HERE, arm["manifest"]) # an absolute path stays as it is
|
||||
return site
|
||||
|
||||
|
||||
def load_sets(site, arm):
|
||||
cfg = site["arms"][arm]
|
||||
man = json.load(open(cfg["manifest"]))
|
||||
for e in man["sets"]:
|
||||
e["arm"] = arm
|
||||
return man["sets"]
|
||||
|
||||
|
||||
def select(site, arms, only, tier):
|
||||
sets = [e for arm in arms for e in load_sets(site, arm)]
|
||||
if only:
|
||||
want = [s.strip() for s in only.split(",") if s.strip()]
|
||||
unknown = [w for w in want if w not in {e["id"] for e in sets}]
|
||||
if unknown:
|
||||
sys.exit(f"not in the {'/'.join(arms)} manifest(s): {', '.join(unknown)}")
|
||||
sets = [e for e in sets if e["id"] in want]
|
||||
if tier:
|
||||
sets = [e for e in sets if tier in e.get("tiers", {})]
|
||||
return sets
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ the binary
|
||||
|
||||
def sha256(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as fh:
|
||||
for block in iter(lambda: fh.read(1 << 20), b""):
|
||||
h.update(block)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def binary_info(path, build_dir):
|
||||
"""Version and git hash as the binary itself prints them, plus the build flags from the
|
||||
CMakeCache of its build tree (given, or found above the binary)."""
|
||||
out = subprocess.run([path], capture_output=True, text=True, timeout=120).stdout
|
||||
m = re.search(r"Version (\S+) \(git (\w+)", out)
|
||||
info = {"version": m.group(1) if m else None, "git": m.group(2) if m else "unknown"}
|
||||
cache = None
|
||||
d = build_dir or os.path.dirname(os.path.realpath(path))
|
||||
for _ in range(4):
|
||||
if os.path.exists(os.path.join(d, "CMakeCache.txt")):
|
||||
cache = os.path.join(d, "CMakeCache.txt")
|
||||
break
|
||||
d = os.path.dirname(d)
|
||||
if cache:
|
||||
vals = {}
|
||||
for line in open(cache, errors="replace"):
|
||||
m = re.match(r"^(CMAKE_BUILD_TYPE|CMAKE_CXX_FLAGS|CMAKE_HOME_DIRECTORY|JFJOCH_USE_CUDA):\w+=(.*)$",
|
||||
line.strip())
|
||||
if m:
|
||||
vals[m.group(1)] = m.group(2)
|
||||
info["cmake_cache"] = cache
|
||||
info["flags"] = (f"{vals.get('CMAKE_BUILD_TYPE', '?')} CXX_FLAGS='{vals.get('CMAKE_CXX_FLAGS', '')}' "
|
||||
f"CUDA={vals.get('JFJOCH_USE_CUDA', '?')}")
|
||||
src = vals.get("CMAKE_HOME_DIRECTORY")
|
||||
if src:
|
||||
info["source"] = src
|
||||
info["source_head"] = git(src, "rev-parse", "HEAD")
|
||||
info["source_dirty"] = bool(git(src, "status", "--porcelain", "--untracked-files=no"))
|
||||
return info
|
||||
|
||||
|
||||
def git(d, *args):
|
||||
r = subprocess.run(["git", "-C", d] + list(args), capture_output=True, text=True)
|
||||
return r.stdout.strip() if r.returncode == 0 else None
|
||||
|
||||
|
||||
def gpu_others():
|
||||
"""Number of processes other than ours on the GPU right now (None if nvidia-smi is absent)."""
|
||||
try:
|
||||
out = subprocess.run(["nvidia-smi", "--query-compute-apps=pid", "--format=csv,noheader"],
|
||||
capture_output=True, text=True, timeout=30).stdout
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
return len([x for x in out.split() if x.strip().isdigit()])
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ running
|
||||
|
||||
def command(binary, e, opts):
|
||||
cmd = [binary, "-o", "p", "--no-export-unmerged"]
|
||||
if opts["threads"]:
|
||||
cmd += ["-N", str(opts["threads"])]
|
||||
ref = e.get("ref") or {}
|
||||
# On an XDS-referenced arm the only thing fixed is XDS's resolution range, so the two programs
|
||||
# merge the same reflections. --unforced leaves rugnux its own cut.
|
||||
if opts["xds_range"] and e.get("reference") == "xds":
|
||||
if ref.get("dmin"):
|
||||
cmd += ["--scaling-high-resolution", f"{ref['dmin']:.3f}"]
|
||||
if ref.get("dmax"):
|
||||
cmd += ["--scaling-low-resolution", f"{ref['dmax']:.3f}"]
|
||||
cmd += opts["extra_args"]
|
||||
cmd.append(e["input_path"])
|
||||
if opts["gpulock"]:
|
||||
cmd = [opts["gpulock"]] + cmd
|
||||
return cmd
|
||||
|
||||
|
||||
def failure_note(log):
|
||||
"""rugnux's own last error message, without the spdlog timestamp and severity prefix."""
|
||||
try:
|
||||
lines = [l.strip() for l in open(log, errors="replace") if l.strip()]
|
||||
except OSError:
|
||||
return ""
|
||||
for l in reversed(lines):
|
||||
if "[error]" in l or "failed" in l.lower():
|
||||
return l.split("] ")[-1][:300]
|
||||
return lines[-1].split("] ")[-1][:300] if lines else ""
|
||||
|
||||
|
||||
def run_one(run_dir, binary, e, opts):
|
||||
wd = os.path.join(run_dir, "work", e["arm"], e["id"])
|
||||
os.makedirs(wd)
|
||||
row = {"set": e["id"], "arm": e["arm"], "tags": e.get("tags", []), "input": e["input_path"],
|
||||
"gpu_others": gpu_others()}
|
||||
if not os.path.exists(e["input_path"]):
|
||||
row.update(score.judge(e, {}, "no input"), exit_code=None, elapsed_s=None, wall_s=None, cmd=None)
|
||||
return row
|
||||
cmd = command(binary, e, opts)
|
||||
row["cmd"] = " ".join(cmd)
|
||||
t0 = time.time()
|
||||
note = ""
|
||||
with open(os.path.join(wd, "run.log"), "w") as log:
|
||||
try:
|
||||
rc = subprocess.run(cmd, cwd=wd, stdout=log, stderr=subprocess.STDOUT,
|
||||
timeout=opts["timeout"]).returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
rc, note = None, "timeout"
|
||||
row["elapsed_s"] = round(time.time() - t0, 1)
|
||||
row["exit_code"] = rc
|
||||
if rc:
|
||||
note = f"exit {rc}: {failure_note(os.path.join(wd, 'run.log'))}"
|
||||
row.update(score.judge(e, score.read_report(os.path.join(wd, "p_report.txt")), note))
|
||||
# rugnux's own WALL_TIME where it wrote one: the runner's clock also counts the wait for a
|
||||
# GPU slot under --gpulock
|
||||
row["wall_s"] = row["rugnux_wall_s"] if row["rugnux_wall_s"] is not None else row["elapsed_s"]
|
||||
return row
|
||||
|
||||
|
||||
def make_read_only(path):
|
||||
for dp, dirs, files in os.walk(path):
|
||||
for fn in files:
|
||||
p = os.path.join(dp, fn)
|
||||
if not os.path.islink(p):
|
||||
os.chmod(p, os.stat(p).st_mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
|
||||
for dp, dirs, files in os.walk(path, topdown=False):
|
||||
os.chmod(dp, os.stat(dp).st_mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
|
||||
|
||||
|
||||
def write_json(path, obj):
|
||||
with open(path + ".tmp", "w") as fh:
|
||||
json.dump(obj, fh, indent=1)
|
||||
os.replace(path + ".tmp", path)
|
||||
|
||||
|
||||
def write_manifest(path, man):
|
||||
"""A manifest with one set per line, so a diff of it reads set by set."""
|
||||
lines = [json.dumps(e) for e in man["sets"]]
|
||||
head = json.dumps({k: v for k, v in man.items() if k != "sets"})[:-1]
|
||||
with open(path, "w") as fh:
|
||||
fh.write(head + ', "sets": [\n ' + ",\n ".join(lines) + "\n]}\n")
|
||||
|
||||
|
||||
def write_report(run_dir, baseline=None, out_dir=None):
|
||||
"""report.md and report.html (report_PRIVATE.* for a private run) into out_dir, by default
|
||||
the run directory itself."""
|
||||
man = json.load(open(os.path.join(run_dir, "manifest.json")))
|
||||
base = os.path.join(out_dir or run_dir, "report_PRIVATE" if man.get("private") else "report")
|
||||
doc = report.build(run_dir, baseline)
|
||||
open(base + ".md", "w").write(doc.markdown())
|
||||
open(base + ".html", "w").write(doc.html())
|
||||
return base + ".html"
|
||||
|
||||
|
||||
def run_sets(site, sets, arms, binary_src, build_dir, label, opts, subset, baseline=None, lock=None):
|
||||
"""Run the given manifest rows into a new run directory. Returns the directory."""
|
||||
private = any(site["arms"][a].get("private") for a in arms)
|
||||
if private and len(arms) > 1:
|
||||
sys.exit("the private arm runs on its own: its results must not share a run or a report")
|
||||
lock_fh = None
|
||||
if lock:
|
||||
os.makedirs(os.path.dirname(lock), exist_ok=True)
|
||||
lock_fh = open(lock, "w")
|
||||
try:
|
||||
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
sys.exit(f"another full battery run holds {lock}; refusing to start a second one")
|
||||
|
||||
info = binary_info(binary_src, build_dir)
|
||||
stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M")
|
||||
run_dir = os.path.join(opts["runs_root"],
|
||||
f"{stamp}_{info['git']}_{label}" + ("_private" if private else ""))
|
||||
os.makedirs(run_dir) # never reuse or overwrite a run directory
|
||||
binary = os.path.join(run_dir, "bin", "rugnux")
|
||||
os.makedirs(os.path.dirname(binary))
|
||||
shutil.copy2(binary_src, binary)
|
||||
info.update(copied_from=os.path.realpath(binary_src), sha256=sha256(binary))
|
||||
|
||||
for e in sets:
|
||||
e["reference"] = site["arms"][e["arm"]]["reference"]
|
||||
e.setdefault("input_path", os.path.join(site["arms"][e["arm"]]["data_root"], e["input"]))
|
||||
man = {"label": label, "arms": arms, "private": private, "subset": subset, "baseline": baseline,
|
||||
"binary": info,
|
||||
"runner_git": git(HERE, "rev-parse", "--short", "HEAD") or "unknown",
|
||||
"runner_dirty": bool(git(HERE, "status", "--porcelain", "--", ".")),
|
||||
"host": socket.gethostname(), "start": datetime.datetime.now().isoformat(timespec="seconds"),
|
||||
"end": None, "complete": False, "options": {k: v for k, v in opts.items() if k != "runs_root"},
|
||||
"gpu_shared": bool(opts["gpulock"]), "sets": sets}
|
||||
write_json(os.path.join(run_dir, "manifest.json"), man)
|
||||
print(f"run directory {run_dir}", flush=True)
|
||||
|
||||
results = []
|
||||
for n, e in enumerate(sets, 1):
|
||||
row = run_one(run_dir, binary, e, opts)
|
||||
results.append(row)
|
||||
if row.get("gpu_others"):
|
||||
man["gpu_shared"] = True
|
||||
write_json(os.path.join(run_dir, "results.json"), results)
|
||||
print(f"[{n}/{len(sets)}] {e['arm']:8s} {e['id']:24s} {row['verdict']:8s} "
|
||||
f"{row.get('sg') or '-':12s} d={report.f(row.get('d_min'))} "
|
||||
f"t={report.f(row.get('wall_s'), '{:.0f}')}s {row.get('reason') or ''}", flush=True)
|
||||
|
||||
man.update(end=datetime.datetime.now().isoformat(timespec="seconds"), complete=True)
|
||||
write_json(os.path.join(run_dir, "manifest.json"), man)
|
||||
print(f"report {write_report(run_dir, baseline)}", flush=True)
|
||||
make_read_only(run_dir)
|
||||
if lock_fh:
|
||||
lock_fh.close()
|
||||
return run_dir
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ subcommands
|
||||
|
||||
def cmd_run(a, site):
|
||||
arms = [x.strip() for x in a.arm.split(",")]
|
||||
for arm in arms:
|
||||
if arm not in site["arms"]:
|
||||
sys.exit(f"unknown arm {arm}; the site config has {', '.join(site['arms'])}")
|
||||
sets = select(site, arms, a.only, a.tier)
|
||||
if not sets:
|
||||
sys.exit("nothing selected")
|
||||
subset = a.only or (f"tier {a.tier}" if a.tier else None)
|
||||
opts = {"threads": a.threads, "xds_range": not a.unforced, "timeout": a.timeout,
|
||||
"gpulock": site["gpulock"] if a.gpulock else None, "extra_args": a.extra.split(),
|
||||
"runs_root": a.runs_root or site["runs_root"]}
|
||||
run_sets(site, sets, arms, a.rugnux, a.build_dir, a.label, opts, subset, a.baseline,
|
||||
lock=None if subset else site["lock_file"])
|
||||
|
||||
|
||||
def cmd_compare(a, site):
|
||||
man_a, res_a = report.load_run(a.run_a)
|
||||
man_b, res_b = report.load_run(a.run_b)
|
||||
rows = report.compare_rows(res_a, res_b)
|
||||
doc = report.Doc(f"{os.path.basename(os.path.normpath(a.run_a))} vs "
|
||||
f"{os.path.basename(os.path.normpath(a.run_b))}")
|
||||
doc.p(f"A: rugnux {man_a['binary'].get('version')} ({man_a['binary']['sha256'][:12]}); "
|
||||
f"B: rugnux {man_b['binary'].get('version')} ({man_b['binary']['sha256'][:12]}). "
|
||||
f"{len(rows)} sets, {sum(1 for r in rows if r[3])} changed beyond noise.")
|
||||
report.compare_table(doc, rows, only_changed=not a.all)
|
||||
print(doc.markdown())
|
||||
if not a.rerun_changed:
|
||||
return
|
||||
# Rerun the changed sets with A's own binary and options: a set that moves again under the
|
||||
# same binary was noise; one that reproduces A was changed by B for real.
|
||||
changed = {k for k, ra, rb, ch in rows if ra and rb and [c for c in ch if c != "time"]}
|
||||
if not changed:
|
||||
print("nothing changed beyond noise; no rerun")
|
||||
return
|
||||
sets = [e for e in man_a["sets"] if (e["arm"], e["id"]) in changed]
|
||||
opts = dict(man_a["options"], runs_root=a.runs_root or site["runs_root"])
|
||||
label = "rerunA-" + man_a["label"]
|
||||
rerun = run_sets(site, sets, man_a["arms"], os.path.join(a.run_a, "bin", "rugnux"), None, label,
|
||||
opts, ",".join(e["id"] for e in sets))
|
||||
_, res_a2 = report.load_run(rerun)
|
||||
again = {k: ch for k, ra, ra2, ch in report.compare_rows(res_a, res_a2) if ra and ra2}
|
||||
doc = report.Doc("Changed sets, rerun with A's binary")
|
||||
rows_out = []
|
||||
for k, ra, rb, ch in rows:
|
||||
if k not in changed:
|
||||
continue
|
||||
noise = [c for c in again.get(k, []) if c != "time"]
|
||||
rows_out.append([k[1], k[0], ", ".join(c for c in ch if c != "time"),
|
||||
", ".join(noise) or "-",
|
||||
"NOISE - A itself moved" if noise else "REAL - A reproduces"])
|
||||
doc.table(["set", "arm", "A -> B", "A -> A again", "reading"], rows_out)
|
||||
doc.p(f"rerun directory: {rerun}")
|
||||
print(doc.markdown())
|
||||
|
||||
|
||||
def cmd_report(a, site):
|
||||
# a finished run directory is read-only, so a re-render goes elsewhere
|
||||
os.makedirs(a.out, exist_ok=True)
|
||||
print(write_report(a.run, a.baseline, a.out))
|
||||
|
||||
|
||||
def cmd_discover(a, site):
|
||||
"""Propose an input for every open-arm directory the manifest does not have yet; with
|
||||
--refresh, also show where today's discovery disagrees with the manifest's (unpinned) input."""
|
||||
cfg = site["arms"]["open"]
|
||||
root = cfg["data_root"]
|
||||
sets = {e["id"]: e for e in load_sets(site, "open")}
|
||||
for d in sorted(os.listdir(root)):
|
||||
if d.startswith(".") or not os.path.isdir(os.path.join(root, d)):
|
||||
continue
|
||||
e = sets.get(d)
|
||||
if e and (not a.refresh or e.get("pinned")):
|
||||
continue
|
||||
path, fmt, note = inputs.find_input(os.path.join(root, d))
|
||||
rel = os.path.relpath(path, root) if path else None
|
||||
if not e:
|
||||
print(f"NEW {d:10s} {rel} [{note}]")
|
||||
elif rel != e["input"]:
|
||||
print(f"DIFF {d:10s} manifest {e['input']}\n {'':10s} discover {rel} [{note}]")
|
||||
for d in sorted(set(sets) - set(os.listdir(root))):
|
||||
print(f"GONE {d:10s} in the manifest, not on disk")
|
||||
|
||||
|
||||
def cmd_refs(a, site):
|
||||
"""Re-read every set's XDS reference from the CORRECT.LP beside its input."""
|
||||
cfg = site["arms"][a.arm]
|
||||
if cfg["reference"] != "xds":
|
||||
sys.exit(f"the {a.arm} arm is not referenced to XDS")
|
||||
man = json.load(open(cfg["manifest"]))
|
||||
for e in man["sets"]:
|
||||
lp = os.path.join(cfg["data_root"], os.path.dirname(e["input"]), "CORRECT.LP")
|
||||
if not os.path.exists(lp):
|
||||
print(f"{e['id']:24s} no CORRECT.LP")
|
||||
continue
|
||||
new = inputs.parse_correct_lp(lp)
|
||||
if new != e.get("ref"):
|
||||
print(f"{e['id']:24s} {e.get('ref')}\n{'':24s} -> {new}")
|
||||
e["ref"] = new
|
||||
if a.write:
|
||||
write_manifest(cfg["manifest"], man)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
||||
ap.add_argument("--site", default=os.environ.get("BATTERY_SITE", DEFAULT_SITE),
|
||||
help="site config (data roots, runs root, private manifest, lock file)")
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
r = sub.add_parser("run")
|
||||
r.add_argument("--rugnux", required=True)
|
||||
r.add_argument("--label", required=True, help="short name, part of the run directory's name")
|
||||
r.add_argument("--arm", default="open,inhouse", help="comma-separated (default: open,inhouse)")
|
||||
r.add_argument("--only", help="comma-separated set ids")
|
||||
r.add_argument("--tier", help="named subset from the manifests, e.g. smoke")
|
||||
r.add_argument("--build-dir", help="build tree whose CMakeCache.txt records the flags")
|
||||
r.add_argument("--gpulock", action="store_true",
|
||||
help="queue each rugnux behind the site's GPU lock (timing is then not a reference)")
|
||||
r.add_argument("--threads", type=int, default=0, help="-N for rugnux (default: its own choice)")
|
||||
r.add_argument("--unforced", action="store_true",
|
||||
help="do not force XDS's resolution range on the XDS-referenced arms")
|
||||
r.add_argument("--timeout", type=int, default=10800)
|
||||
r.add_argument("--extra", default="", help="extra rugnux arguments, applied to every set")
|
||||
r.add_argument("--baseline", help="earlier run directory to report the delta against")
|
||||
r.add_argument("--runs-root")
|
||||
|
||||
c = sub.add_parser("compare")
|
||||
c.add_argument("run_a")
|
||||
c.add_argument("run_b")
|
||||
c.add_argument("--all", action="store_true", help="list unchanged sets too")
|
||||
c.add_argument("--rerun-changed", action="store_true",
|
||||
help="rerun the changed sets with A's saved binary to separate real change from noise")
|
||||
c.add_argument("--runs-root")
|
||||
|
||||
p = sub.add_parser("report")
|
||||
p.add_argument("run")
|
||||
p.add_argument("--baseline")
|
||||
p.add_argument("--out", required=True, help="directory for the re-rendered report")
|
||||
|
||||
d = sub.add_parser("discover")
|
||||
d.add_argument("--refresh", action="store_true")
|
||||
|
||||
f = sub.add_parser("refs")
|
||||
f.add_argument("--arm", required=True)
|
||||
f.add_argument("--write", action="store_true")
|
||||
|
||||
a = ap.parse_args()
|
||||
site = load_site(a.site)
|
||||
{"run": cmd_run, "compare": cmd_compare, "report": cmd_report,
|
||||
"discover": cmd_discover, "refs": cmd_refs}[a.cmd](a, site)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,38 @@
|
||||
{"arm": "inhouse", "sets": [
|
||||
{"id": "E08_lyso_2_006", "input": "E08_lyso_2_006/lyso_2_006_master.h5", "ref": {"sgno": 92, "cell": [76.879, 76.879, 36.961, 90.0, 90.0, 90.0], "anomalous": true, "isa": 20.75, "completeness": 99.3, "r_meas": 0.182, "cc_half": 0.995, "multiplicity": 3.51, "dmin": 1.966, "dmax": 50.0}, "tags": ["h5", "lysozyme"]},
|
||||
{"id": "E08_lyso_2_007", "input": "E08_lyso_2_007/lyso_2_007_master.h5", "ref": {"sgno": 92, "cell": [76.904, 76.904, 36.976, 90.0, 90.0, 90.0], "anomalous": true, "isa": 18.14, "completeness": 99.3, "r_meas": 0.18, "cc_half": 0.994, "multiplicity": 3.51, "dmin": 1.973, "dmax": 50.0}, "tags": ["h5", "lysozyme"]},
|
||||
{"id": "lyso_2", "input": "lyso_2/lyso_2_002_master.h5", "ref": {"sgno": 92, "cell": [78.235, 78.235, 37.798, 90.0, 90.0, 90.0], "anomalous": true, "isa": 16.56, "completeness": 99.9, "r_meas": 0.239, "cc_half": 0.998, "multiplicity": 10.97, "dmin": 1.264, "dmax": 50.0}, "tags": ["h5", "lysozyme"]},
|
||||
{"id": "lyso_6_5keV", "input": "lyso_6_5keV/lyso_6_001_master.h5", "ref": {"sgno": 92, "cell": [78.225, 78.225, 37.897, 90.0, 90.0, 90.0], "anomalous": true, "isa": 19.36, "completeness": 86.4, "r_meas": 0.064, "cc_half": 0.998, "multiplicity": 8.85, "dmin": 2.45, "dmax": 50.0}, "tags": ["h5", "lysozyme", "long-wavelength"]},
|
||||
{"id": "lysoC_14_f46f0a", "input": "lysoC_14_f46f0a/lysoC_14_56b5b9_master.h5", "ref": {"sgno": 89, "cell": [78.387, 78.387, 37.427, 90.0, 90.0, 90.0], "anomalous": true, "isa": 23.33, "completeness": 100.0, "r_meas": 0.219, "cc_half": 0.998, "multiplicity": 11.86, "dmin": 1.431, "dmax": 50.0}, "tags": ["h5", "lysozyme"]},
|
||||
{"id": "lyso_ref", "input": "lyso_ref/hewl_35_001_master.h5", "ref": {"sgno": 89, "cell": [78.091, 78.091, 37.704, 90.0, 90.0, 90.0], "anomalous": true, "isa": 28.31, "completeness": 100.0, "r_meas": 0.045, "cc_half": 1.0, "multiplicity": 13.76, "dmin": 1.2, "dmax": 50.0}, "tags": ["h5", "lysozyme"], "tiers": {"smoke": "lysozyme reference set"}},
|
||||
{"id": "lyso_strong", "input": "lyso_strong/lyso_5_f6ac7e_master.h5", "ref": {"sgno": 92, "cell": [77.769, 77.769, 37.442, 90.0, 90.0, 90.0], "anomalous": false, "isa": 8.63, "completeness": 65.8, "r_meas": 0.112, "cc_half": 0.998, "multiplicity": 11.46, "dmin": 1.18, "dmax": 999.0}, "tags": ["h5", "lysozyme"]},
|
||||
{"id": "lyso_half_image", "input": "lyso_half_image/lyso3_7_b81d83_master.h5", "ref": {"sgno": 1, "cell": [37.267, 79.125, 81.145, 87.995, 90.636, 90.239], "anomalous": true, "isa": 1.87, "completeness": 43.2, "r_meas": 3.17, "cc_half": 0.179, "multiplicity": 1.44, "dmin": 0.8, "dmax": 50.0}, "tags": ["h5", "lysozyme"]},
|
||||
{"id": "thau_16keV", "input": "thau_16keV/lyso_4_d2bb9b_master.h5", "ref": {"sgno": 92, "cell": [57.958, 57.958, 150.547, 90.0, 90.0, 90.0], "anomalous": true, "isa": 44.79, "completeness": 94.3, "r_meas": 0.073, "cc_half": 1.0, "multiplicity": 9.76, "dmin": 1.3, "dmax": 50.0}, "tags": ["h5", "thaumatin"]},
|
||||
{"id": "Thau_9", "input": "Thau_9/Thau_9_002_master.h5", "ref": {"sgno": 92, "cell": [58.488, 58.488, 150.788, 90.0, 90.0, 90.0], "anomalous": true, "isa": 9.15, "completeness": 99.0, "r_meas": 0.23, "cc_half": 0.997, "multiplicity": 11.37, "dmin": 2.197, "dmax": 50.0}, "tags": ["h5", "thaumatin"]},
|
||||
{"id": "thau_BL1A_1p9A", "input": "thau_BL1A_1p9A/JTH14_1p9_50ds_90T_k0_JF4M_01_50001_master.h5", "ref": {"sgno": 89, "cell": [58.031, 58.031, 150.577, 90.0, 90.0, 90.0], "anomalous": true, "isa": 34.46, "completeness": 91.8, "r_meas": 0.072, "cc_half": 0.999, "multiplicity": 7.95, "dmin": 1.78, "dmax": 100.0}, "tags": ["h5", "thaumatin", "long-wavelength"]},
|
||||
{"id": "thau_BL1A_2p7A", "input": "thau_BL1A_2p7A/JTH14_2p7_50ds_19T_k0_JF4M_01_50001_master.h5", "ref": {"sgno": 89, "cell": [58.007, 58.007, 150.511, 90.0, 90.0, 90.0], "anomalous": true, "isa": 35.64, "completeness": 92.0, "r_meas": 0.062, "cc_half": 0.998, "multiplicity": 7.94, "dmin": 2.53, "dmax": 100.0}, "tags": ["h5", "thaumatin", "long-wavelength"]},
|
||||
{"id": "thau_BL1A_3p3A", "input": "thau_BL1A_3p3A/JTH14_3p3_50ds_12T_k0_JF4M_01_50001_master.h5", "ref": {"sgno": 89, "cell": [57.895, 57.895, 150.213, 90.0, 90.0, 90.0], "anomalous": true, "isa": 30.8, "completeness": 92.2, "r_meas": 0.057, "cc_half": 0.997, "multiplicity": 7.92, "dmin": 3.1, "dmax": 100.0}, "tags": ["h5", "thaumatin", "long-wavelength"], "tiers": {"smoke": "long wavelength (3.3 A), KEK BL-1A thaumatin"}},
|
||||
{"id": "thau_injection", "input": "thau_injection/lyso_13_0c7e7a_master.h5", "ref": {"sgno": 92, "cell": [57.922, 57.922, 150.678, 90.0, 90.0, 90.0], "anomalous": true, "isa": 36.23, "completeness": 86.2, "r_meas": 0.037, "cc_half": 1.0, "multiplicity": 9.96, "dmin": 1.28, "dmax": 50.0}, "tags": ["h5", "thaumatin"]},
|
||||
{"id": "thau_pink", "input": "thau_pink/Thau-Thaumatin_2_master.h5", "ref": {"sgno": 92, "cell": [58.784, 58.784, 152.226, 90.0, 90.0, 90.0], "anomalous": true, "isa": 21.18, "completeness": 98.5, "r_meas": 0.08, "cc_half": 0.999, "multiplicity": 9.38, "dmin": 1.4, "dmax": 50.0}, "tags": ["h5", "thaumatin", "pink-beam"], "tiers": {"smoke": "pink beam"}},
|
||||
{"id": "Ins_H_2_fa9841", "input": "Ins_H_2_fa9841/Ins_H_2_fa9841_master.h5", "ref": {"sgno": 146, "cell": [81.393, 81.393, 33.272, 90.0, 90.0, 120.0], "anomalous": true, "isa": 6.82, "completeness": 95.0, "r_meas": 0.118, "cc_half": 0.988, "multiplicity": 3.61, "dmin": 1.455, "dmax": 50.0}, "tags": ["h5", "insulin"]},
|
||||
{"id": "Ins_H_3_d05684", "input": "Ins_H_3_d05684/Ins_H_3_d05684_master.h5", "ref": {"sgno": 146, "cell": [81.378, 81.378, 33.269, 90.0, 90.0, 120.0], "anomalous": true, "isa": 17.69, "completeness": 97.4, "r_meas": 0.068, "cc_half": 0.998, "multiplicity": 3.52, "dmin": 1.544, "dmax": 50.0}, "tags": ["h5", "insulin"], "tiers": {"smoke": "rhombohedral insulin (H3)"}},
|
||||
{"id": "Ins_I_16", "input": "Ins_I_16/Ins_16_776a92_master.h5", "ref": {"sgno": 197, "cell": [77.085, 77.085, 77.085, 90.0, 90.0, 90.0], "anomalous": false, "isa": 18.91, "completeness": 92.4, "r_meas": 0.401, "cc_half": 0.999, "multiplicity": 29.04, "dmin": 1.08, "dmax": 999.0}, "tags": ["h5", "insulin"]},
|
||||
{"id": "Ins_I_2_1a530d", "input": "Ins_I_2_1a530d/Ins_I_2_1a530d_master.h5", "ref": {"sgno": 197, "cell": [77.582, 77.582, 77.582, 90.0, 90.0, 90.0], "anomalous": true, "isa": 4.15, "completeness": 90.8, "r_meas": 0.22, "cc_half": 0.998, "multiplicity": 13.33, "dmin": 1.3, "dmax": 50.0}, "tags": ["h5", "insulin"]},
|
||||
{"id": "Ins_I_3_5e90e3", "input": "Ins_I_3_5e90e3/Ins_I_3_5e90e3_master.h5", "ref": {"sgno": 197, "cell": [77.52, 77.52, 77.52, 90.0, 90.0, 90.0], "anomalous": true, "isa": 25.08, "completeness": 100.0, "r_meas": 0.349, "cc_half": 0.998, "multiplicity": 13.82, "dmin": 1.621, "dmax": 50.0}, "tags": ["h5", "insulin"]},
|
||||
{"id": "Ins_I_6_13keV", "input": "Ins_I_6_13keV/Ins_I_6_001_master.h5", "ref": {"sgno": 197, "cell": [77.468, 77.468, 77.468, 90.0, 90.0, 90.0], "anomalous": true, "isa": 18.83, "completeness": 100.0, "r_meas": 0.279, "cc_half": 0.998, "multiplicity": 17.21, "dmin": 1.635, "dmax": 50.0}, "tags": ["h5", "insulin"]},
|
||||
{"id": "Ins_I_6_5keV", "input": "Ins_I_6_5keV/Ins_I_6_006_master.h5", "ref": {"sgno": 197, "cell": [77.785, 77.785, 77.785, 90.0, 90.0, 90.0], "anomalous": true, "isa": 17.54, "completeness": 91.7, "r_meas": 0.073, "cc_half": 0.999, "multiplicity": 11.17, "dmin": 2.45, "dmax": 50.0}, "tags": ["h5", "insulin", "long-wavelength"]},
|
||||
{"id": "Ins_I_6_6keV", "input": "Ins_I_6_6keV/Ins_I_6_004_master.h5", "ref": {"sgno": 197, "cell": [77.71, 77.71, 77.71, 90.0, 90.0, 90.0], "anomalous": true, "isa": 17.95, "completeness": 92.4, "r_meas": 0.065, "cc_half": 0.999, "multiplicity": 12.42, "dmin": 2.04, "dmax": 50.0}, "tags": ["h5", "insulin", "long-wavelength"]},
|
||||
{"id": "Ins_I_7_5keV", "input": "Ins_I_7_5keV/Ins_I_7_001_master.h5", "ref": {"sgno": 197, "cell": [77.694, 77.694, 77.694, 90.0, 90.0, 90.0], "anomalous": true, "isa": 20.01, "completeness": 91.8, "r_meas": 0.076, "cc_half": 0.999, "multiplicity": 12.73, "dmin": 2.45, "dmax": 50.0}, "tags": ["h5", "insulin", "long-wavelength"], "tiers": {"smoke": "cubic insulin at 5 keV (long wavelength), fast"}},
|
||||
{"id": "cytC_10", "input": "cytC_10/cytC_10_002_master.h5", "ref": {"sgno": 152, "cell": [83.659, 83.659, 86.011, 90.0, 90.0, 120.0], "anomalous": true, "isa": 31.75, "completeness": 99.8, "r_meas": 0.232, "cc_half": 0.999, "multiplicity": 10.65, "dmin": 2.039, "dmax": 50.0}, "tags": ["h5", "cytochrome-c"]},
|
||||
{"id": "cytC_2_0770b5", "input": "cytC_2_0770b5/cytC_2_0770b5_master.h5", "ref": {"sgno": 152, "cell": [83.836, 83.836, 86.733, 90.0, 90.0, 120.0], "anomalous": true, "isa": 24.49, "completeness": 99.9, "r_meas": 0.084, "cc_half": 0.999, "multiplicity": 7.67, "dmin": 1.875, "dmax": 50.0}, "tags": ["h5", "cytochrome-c"]},
|
||||
{"id": "cytC_3_874285", "input": "cytC_3_874285/cytC_3_874285_master.h5", "ref": {"sgno": 152, "cell": [83.719, 83.719, 88.617, 90.0, 90.0, 120.0], "anomalous": true, "isa": 26.98, "completeness": 99.9, "r_meas": 0.097, "cc_half": 0.999, "multiplicity": 7.82, "dmin": 1.69, "dmax": 50.0}, "tags": ["h5", "cytochrome-c"]},
|
||||
{"id": "MyoB_13", "input": "MyoB_13/MyoB_13_110ee0_master.h5", "ref": {"sgno": 4, "cell": [35.229, 28.461, 63.076, 90.0, 106.504, 90.0], "anomalous": true, "isa": 5.21, "completeness": 97.7, "r_meas": 0.247, "cc_half": 0.965, "multiplicity": 3.29, "dmin": 1.742, "dmax": 50.0}, "tags": ["h5", "myoglobin"]},
|
||||
{"id": "MyoB_9_446b70", "input": "MyoB_9_446b70/MyoB_9_446b70_master.h5", "ref": {"sgno": 3, "cell": [35.182, 28.544, 62.83, 90.0, 106.138, 90.0], "anomalous": true, "isa": 7.59, "completeness": 99.4, "r_meas": 0.232, "cc_half": 0.987, "multiplicity": 3.47, "dmin": 1.422, "dmax": 50.0}, "tags": ["h5", "myoglobin"], "tiers": {"smoke": "monoclinic myoglobin, screw not arbitrable by XDS"}},
|
||||
{"id": "MyoB2-1", "input": "MyoB2-1/MyoB2-1_75979a_master.h5", "ref": {"sgno": 1, "cell": [28.705, 34.796, 64.373, 74.314, 89.879, 90.057], "anomalous": false, "isa": 5.54, "completeness": 74.2, "r_meas": 0.493, "cc_half": 0.966, "multiplicity": 2.87, "dmin": 1.495, "dmax": 50.0}, "tags": ["h5", "myoglobin", "twin"], "tiers": {"smoke": "known hard: triclinic twin"}},
|
||||
{"id": "MyoB2-2", "input": "MyoB2-2/MyoB2-2_5ddd7d_master.h5", "ref": {"sgno": 3, "cell": [35.576, 28.811, 63.529, 90.0, 105.593, 90.0], "anomalous": false, "isa": 2.19, "completeness": 58.0, "r_meas": 1.106, "cc_half": 0.655, "multiplicity": 4.71, "dmin": 0.99, "dmax": 999.0}, "tags": ["h5", "myoglobin", "twin"]},
|
||||
{"id": "MyoB2-4", "input": "MyoB2-4/MyoB2-4_079740_master.h5", "ref": {"sgno": 3, "cell": [35.39, 28.783, 63.626, 90.0, 105.544, 90.0], "anomalous": false, "isa": 12.41, "completeness": 77.7, "r_meas": 0.577, "cc_half": 0.984, "multiplicity": 4.38, "dmin": 1.506, "dmax": 50.0}, "tags": ["h5", "myoglobin", "twin"]},
|
||||
{"id": "MyoB2-5", "input": "MyoB2-5/MyoB2-5_7cef9c_master.h5", "ref": {"sgno": 1, "cell": [43.226, 67.346, 121.692, 100.15, 89.855, 96.62], "anomalous": false, "isa": 1.24, "completeness": 11.8, "r_meas": 1.527, "cc_half": 0.125, "multiplicity": 1.41, "dmin": 1.11, "dmax": 999.0}, "tags": ["h5", "myoglobin", "twin"]},
|
||||
{"id": "onlyice", "input": "onlyice/onlyice_master.h5", "expect": "no_lattice", "tags": ["h5", "control"], "tiers": {"smoke": "no-crystal control (ice rings only): must refuse"}},
|
||||
{"id": "nothing_3400ac", "input": "nothing/test-28_3400ac_master.h5", "expect": "no_lattice", "tags": ["h5", "control"]},
|
||||
{"id": "nothing_9bf604", "input": "nothing/test-28_9bf604_master.h5", "expect": "no_lattice", "tags": ["h5", "control"]}
|
||||
]}
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Finding a dataset's input file, and reading an XDS CORRECT.LP as a reference.
|
||||
|
||||
Both run only when a manifest is (re)built - `battery.py discover` and `battery.py refs`. A battery
|
||||
run itself never searches: it runs the input the manifest names, so two runs of one manifest always
|
||||
process the same files.
|
||||
"""
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
|
||||
# XDS writes calibration images beside the frames; none of them is a sweep.
|
||||
XDS_AUX = ("-CORRECTIONS", "BKGINIT", "BKGPIX", "BLANK", "GAIN", "DECAY", "ABS.", "MODPIX", "FRAME.")
|
||||
|
||||
|
||||
def master_frames(path):
|
||||
"""Number of frames a NXmx master describes.
|
||||
|
||||
A sweep is chosen by frame count, never by file size: a master with a few embedded screening
|
||||
frames can outweigh the master of a real sweep whose frames sit in external data files."""
|
||||
import h5py
|
||||
try:
|
||||
with h5py.File(path, "r") as h:
|
||||
det = "entry/instrument/detector/detectorSpecific"
|
||||
if f"{det}/nimages" in h:
|
||||
n = int(h[f"{det}/nimages"][()])
|
||||
if f"{det}/ntrigger" in h:
|
||||
n *= max(1, int(h[f"{det}/ntrigger"][()]))
|
||||
return n
|
||||
omega = "entry/sample/goniometer/omega"
|
||||
if omega in h and h[omega].shape:
|
||||
return int(h[omega].shape[0])
|
||||
n = 0
|
||||
for k in h.get("entry/data", {}):
|
||||
try:
|
||||
n += int(h["entry/data"][k].shape[0])
|
||||
except (KeyError, OSError): # an external link whose file is missing
|
||||
pass
|
||||
return n
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def detector_frame(path):
|
||||
"""'marCCD' or 'SMV' if the file's header says so, else None. These formats have no reliable
|
||||
extension (.mccd, .img, .001, ...; .img is SMV at one site and marCCD at the next)."""
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
head = fh.read(64)
|
||||
if head[:2] in (b"II", b"MM"):
|
||||
fh.seek(1028)
|
||||
return "marCCD" if fh.read(3) == b"MMX" else None
|
||||
if head[:1] == b"{" and b"HEADER_BYTES" in head:
|
||||
return "SMV"
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def find_input(d):
|
||||
"""(input file, format, note) for the largest sweep under directory d, or (None, None, why).
|
||||
|
||||
Frames are grouped per directory by template (the last run of digits in the name), and the
|
||||
biggest group by frame count wins; a tie goes to the alphabetically first. Groups are never
|
||||
merged across directories - two crystals can have identically named frames side by side."""
|
||||
masters = [m for m in glob.glob(f"{d}/**/*master*.h5", recursive=True)
|
||||
if not os.path.basename(m).startswith("._")]
|
||||
if not masters:
|
||||
# Diamond writes a NeXus .nxs master beside _NNNNNN.h5 data files
|
||||
masters = [m for m in glob.glob(f"{d}/**/*.nxs", recursive=True)
|
||||
if not os.path.basename(m).startswith("._") and "meta" not in os.path.basename(m)]
|
||||
if masters:
|
||||
best = min(masters, key=lambda m: (-master_frames(m), m))
|
||||
return best, "nxs" if best.endswith(".nxs") else "h5", \
|
||||
f"master, {master_frames(best)} frames, {len(masters)} master(s)"
|
||||
|
||||
groups = {}
|
||||
for p in glob.glob(f"{d}/**/*", recursive=True):
|
||||
name = os.path.basename(p)
|
||||
if not os.path.isfile(p) or name.startswith("._") or any(a in name.upper() for a in XDS_AUX):
|
||||
continue
|
||||
key = (os.path.dirname(p), re.sub(r"\d+(?=\D*$)", "#", name))
|
||||
groups.setdefault(key, []).append(p)
|
||||
for key, files in sorted(groups.items(), key=lambda kv: (-len(kv[1]), kv[0])):
|
||||
if len(files) < 10:
|
||||
break
|
||||
files.sort()
|
||||
if files[0].endswith((".cbf", ".cbf.gz")):
|
||||
fmt = "cbf"
|
||||
else:
|
||||
fmt = detector_frame(files[0])
|
||||
if fmt:
|
||||
return files[0], fmt, f'{fmt} "{key[1]}" x{len(files)} of {len(groups)} group(s)'
|
||||
return None, None, "no readable sweep found"
|
||||
|
||||
|
||||
def input_format(path):
|
||||
"""Format tag of an input file named in a manifest."""
|
||||
if path.endswith(".h5"):
|
||||
return "h5"
|
||||
if path.endswith(".nxs"):
|
||||
return "nxs"
|
||||
if path.endswith((".cbf", ".cbf.gz")):
|
||||
return "cbf"
|
||||
return detector_frame(path) or "other"
|
||||
|
||||
|
||||
def parse_correct_lp(path):
|
||||
"""Reference values from an XDS CORRECT.LP: the space group and cell CORRECT refined, the
|
||||
resolution range it merged, and its overall merging statistics.
|
||||
|
||||
CORRECT.LP states the group and cell twice - first as the INPUT to INTEGRATE (routinely P1),
|
||||
then as CORRECT's own answer. The last SPACE_GROUP_NUMBER and the refined UNIT CELL PARAMETERS
|
||||
are the ones the statistics were computed in."""
|
||||
txt = open(path, errors="replace").read()
|
||||
ref = {}
|
||||
sgs = re.findall(r"SPACE_GROUP_NUMBER=\s*(\d+)", txt)
|
||||
ref["sgno"] = int(sgs[-1]) if sgs else None
|
||||
cells = re.findall(r"UNIT CELL PARAMETERS\s+([\d.\s]+)", txt)
|
||||
if cells:
|
||||
ref["cell"] = [float(x) for x in cells[-1].split()[:6]]
|
||||
else:
|
||||
m = re.findall(r"UNIT_CELL_CONSTANTS=\s*([\d.\s]+)", txt)
|
||||
ref["cell"] = [float(x) for x in m[-1].split()[:6]] if m else None
|
||||
m = re.search(r"FRIEDEL'S_LAW=\s*(TRUE|FALSE)", txt)
|
||||
ref["anomalous"] = bool(m and m.group(1) == "FALSE")
|
||||
m = re.search(r"^\s*a\s+b\s+ISa\s*\n\s*[\d.Ee+-]+\s+[\d.Ee+-]+\s+([\d.]+)", txt, re.M)
|
||||
ref["isa"] = float(m.group(1)) if m else None
|
||||
|
||||
# the last "... AS FUNCTION OF RESOLUTION" table; shells from low to high resolution, then total
|
||||
seg = txt[txt.rfind("AS FUNCTION OF RESOLUTION"):]
|
||||
num = lambda x: float(x.rstrip("*%"))
|
||||
shells, total = [], None
|
||||
for line in seg.splitlines():
|
||||
f = line.split()
|
||||
if f and f[0] == "total" and len(f) >= 11:
|
||||
total = f
|
||||
break
|
||||
if f and re.match(r"^\d+\.\d+$", f[0]) and len(f) >= 11:
|
||||
shells.append(f)
|
||||
if total:
|
||||
ref["completeness"] = num(total[4])
|
||||
ref["r_meas"] = round(num(total[9]) / 100, 4)
|
||||
ref["cc_half"] = round(num(total[10]) / 100, 4)
|
||||
ref["multiplicity"] = round(int(total[1]) / int(total[2]), 2) if int(total[2]) else None
|
||||
|
||||
# The resolution range to hand to rugnux so both programs merge the same reflections: the
|
||||
# explicit INCLUDE_RESOLUTION_RANGE, or - when XDS.INP left the high limit at 0.0, "the whole
|
||||
# detector" - the finest shell actually tabulated.
|
||||
m = re.search(r"INCLUDE_RESOLUTION_RANGE=\s*([\d.]+)\s+([\d.]+)", txt)
|
||||
include_low, include_high = (float(m.group(1)), float(m.group(2))) if m else (0.0, 0.0)
|
||||
table_high = float(shells[-1][0]) if shells else 0.0
|
||||
ref["dmin"] = (include_high if include_high > 0 else table_high) or None
|
||||
ref["dmax"] = include_low or None
|
||||
return ref
|
||||
@@ -0,0 +1,153 @@
|
||||
{"arm": "open", "sets": [
|
||||
{"id": "11if", "input": "11if/BopeA_18500_a_B2-PEF_11if/data/PSL-1503_13949_master.h5", "ref": {"sg": "P 43", "sgno": 78, "cell": [51.14, 51.14, 71.92, 90.0, 90.0, 90.0], "dmin": 1.51}, "tags": ["h5", "tetragonal"], "tiers": {"smoke": "Eiger NXmx from another facility, 4-fold screw"}},
|
||||
{"id": "36gk", "input": "36gk/CLS-0074_5-3_36GK/data/CLS-0074_5-3_master.h5", "ref": {"sg": "I 2 2 2", "sgno": 23, "cell": [120.58, 189.49, 199.69, 90.0, 90.0, 90.0], "dmin": 2.28}, "tags": ["h5", "orthorhombic"]},
|
||||
{"id": "3inp", "input": "3inp/IDP02542_3inp/data/idp02542b.001", "ref": {"sg": "F 41 3 2", "sgno": 210, "cell": [224.08, 224.08, 224.08, 90.0, 90.0, 90.0], "dmin": 2.05}, "tags": ["marCCD", "cubic"]},
|
||||
{"id": "3ky7", "input": "3ky7/IDP90258_3ky7/data/idp90258f.001", "ref": {"sg": "P 43 3 2", "sgno": 212, "cell": [125.176, 125.176, 125.176, 90.0, 90.0, 90.0], "dmin": 2.35}, "tags": ["marCCD", "cubic"]},
|
||||
{"id": "5ebi", "input": "5ebi/dna-rna-chimera_Ba_high_2_001.img", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [35.72, 44.1, 35.72, 90.0, 119.98, 90.0], "dmin": 1.09}, "pinned": true, "tags": ["marCCD", "monoclinic", "twin"]},
|
||||
{"id": "5epe", "input": "5epe/030805_5epe/data/E1_7_set.001", "ref": {"sg": "F 2 3", "sgno": 196, "cell": [157.536, 157.536, 157.536, 90.0, 90.0, 90.0], "dmin": 1.9}, "tags": ["marCCD", "cubic"]},
|
||||
{"id": "5f6m", "input": "5f6m/crystal3_00001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [54.81, 58.51, 67.42, 90.0, 90.0, 90.0], "dmin": 1.1}, "tags": ["cbf", "orthorhombic", "diffuse"]},
|
||||
{"id": "5j23", "input": "5j23/012132_5j23/data/XZ2_set.001", "ref": {"sg": "H 3", "sgno": 146, "cell": [175.822, 175.822, 136.839, 90.0, 90.0, 120.0], "dmin": 2.3}, "tags": ["marCCD", "trigonal", "twin"]},
|
||||
{"id": "5jvn", "input": "5jvn/5jvn/data/PPDK-AV2820_w1_3_0001.cbf", "ref": {"sg": "P 6 2 2", "sgno": 177, "cell": [249.43, 249.43, 84.06, 90.0, 90.0, 120.0], "dmin": 2.9}, "tags": ["cbf", "hexagonal"]},
|
||||
{"id": "5lzl", "input": "5lzl/pcalad/alad-levoil6_M3S15_1_0001.cbf", "ref": {"sg": "P 31 2 1", "sgno": 152, "cell": [205.561, 205.561, 199.171, 90.0, 90.0, 120.0], "dmin": 3.47}, "tags": ["cbf", "trigonal"]},
|
||||
{"id": "5m17", "input": "5m17/BxGH99_WTDD_HA00AG2801_2_0001.cbf", "ref": {"sg": "I 4", "sgno": 79, "cell": [108.576, 108.576, 67.746, 90.0, 90.0, 90.0], "dmin": 1.03}, "tags": ["cbf", "tetragonal"]},
|
||||
{"id": "5nw5", "input": "5nw5/5NW5_1_00001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [92.14, 169.8, 390.16, 90.0, 90.0, 90.0], "dmin": 6.502}, "tags": ["cbf", "orthorhombic", "low-resolution"]},
|
||||
{"id": "5reo", "input": "5reo/cbf/Mpro-x0752_1_0001.cbf", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [112.39, 52.59, 44.38, 90.0, 103.04, 90.0], "dmin": 1.88}, "tags": ["cbf", "monoclinic"], "tiers": {"smoke": "fastest set (10 s): miniCBF, monoclinic"}},
|
||||
{"id": "5src", "input": "5src/5src/data/FRS004_15_1_00001.cbf", "ref": {"sg": "P 43", "sgno": 78, "cell": [88.68, 88.68, 39.23, 90.0, 90.0, 90.0], "dmin": 1.05}, "tags": ["cbf", "tetragonal"]},
|
||||
{"id": "6fid", "input": "6fid/Trypsin_x1_align_2_0001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [59.947, 64.107, 69.694, 90.0, 90.0, 90.0], "dmin": 2.2}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "6fvz", "input": "6fvz/maox215_6fvz/data/maox215_w1_1_0001.cbf", "ref": {"sg": "C 2 2 2", "sgno": 21, "cell": [131.182, 222.753, 86.482, 90.0, 90.0, 90.0], "dmin": 1.8}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "6fwc", "input": "6fwc/maox225_6fwc/data/maox225_1_00001.cbf", "ref": {"sg": "C 2 2 2", "sgno": 21, "cell": [131.728, 222.051, 86.293, 90.0, 90.0, 90.0], "dmin": 1.7}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "6h2p", "input": "6h2p/ProlMut1MnCAC_6h2p/data/Prol_Mut1_Mn_X_pk1_2_0001.cbf", "ref": {"sg": "C 2 2 21", "sgno": 20, "cell": [103.453, 107.075, 216.543, 90.0, 90.0, 90.0], "dmin": 1.479}, "tags": ["cbf", "orthorhombic", "long-wavelength"]},
|
||||
{"id": "6h5t", "input": "6h5t/6h5t/data/SH3-1_x2_1_001.img", "ref": {"sg": "I 4 2 2", "sgno": 97, "cell": [86.792, 86.792, 141.768, 90.0, 90.0, 90.0], "dmin": 1.689}, "tags": ["marCCD", "tetragonal"]},
|
||||
{"id": "6hv2", "input": "6hv2/6hv2/data/coll_1_master.h5", "ref": {"sg": "P 61 2 2", "sgno": 178, "cell": [68.928, 68.928, 133.563, 90.0, 90.0, 120.0], "dmin": 1.709}, "tags": ["h5", "hexagonal"]},
|
||||
{"id": "6hwj", "input": "6hwj/bg3006_2_0001.cbf", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [59.812, 96.07, 80.252, 90.0, 106.69, 90.0], "dmin": 1.979}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "6i3j", "input": "6i3j/6i3j/data/BF19_go_1_0001.img", "ref": {"sg": "F 2 2 2", "sgno": 22, "cell": [134.382, 203.846, 226.744, 90.0, 90.0, 90.0], "dmin": 2.59}, "tags": ["marCCD", "orthorhombic"]},
|
||||
{"id": "6iu5", "input": "6iu5/6iu5_VIT1_MBD_Zn/peak/peak_000001.cbf", "ref": {"sg": "P 31", "sgno": 144, "cell": [84.942, 84.942, 98.19, 90.0, 90.0, 120.0], "dmin": 2.25}, "pinned": true, "tags": ["cbf", "trigonal", "twin"]},
|
||||
{"id": "6iu6", "input": "6iu6/6iu6_VIT1_MBD_Ni/peak/peak_000001.cbf", "ref": {"sg": "P 31", "sgno": 144, "cell": [84.744, 84.744, 97.398, 90.0, 90.0, 120.0], "dmin": 2.9}, "pinned": true, "tags": ["cbf", "trigonal", "twin"]},
|
||||
{"id": "6iu8", "input": "6iu8/6iu8_VIT1_MBD_Co/low/low_000001.cbf", "ref": {"sg": "P 31", "sgno": 144, "cell": [85.503, 85.503, 98.355, 90.0, 90.0, 120.0], "dmin": 2.7}, "tags": ["cbf", "trigonal"]},
|
||||
{"id": "6iu9", "input": "6iu9/6iu9_VIT1_MBD_Fe/peak/data01_000001.cbf", "ref": {"sg": "P 31", "sgno": 144, "cell": [85.321, 85.321, 97.573, 90.0, 90.0, 120.0], "dmin": 3.0}, "pinned": true, "tags": ["cbf", "trigonal", "twin"]},
|
||||
{"id": "6jgi", "input": "6jgi/6jgi/data/00001.img", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [50.857, 62.403, 69.172, 90.0, 90.0, 90.0], "dmin": 0.85}, "tags": ["marCCD", "orthorhombic"]},
|
||||
{"id": "6jgj", "input": "6jgj/6jgj/data/000001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [50.87, 62.34, 68.85, 90.0, 90.0, 90.0], "dmin": 0.77}, "tags": ["cbf", "orthorhombic", "cdte"]},
|
||||
{"id": "6moj", "input": "6moj/A9_1_00001.cbf", "ref": {"sg": "I 41 2 2", "sgno": 98, "cell": [130.418, 130.418, 293.453, 90.0, 90.0, 90.0], "dmin": 2.431}, "tags": ["cbf", "tetragonal"]},
|
||||
{"id": "6o2h", "input": "6o2h/lys_nitr_10_1_0001.cbf", "ref": {"sg": "P 1", "sgno": 1, "cell": [27.424, 32.134, 34.513, 88.657, 108.46, 111.877], "dmin": 1.212}, "tags": ["cbf", "triclinic", "diffuse"]},
|
||||
{"id": "6oel", "input": "6oel/1449-8_3_001.img", "ref": {"sg": "F 41 3 2", "sgno": 210, "cell": [328.1, 328.1, 328.1, 90.0, 90.0, 90.0], "dmin": 3.1}, "tags": ["SMV", "cubic"], "tiers": {"smoke": "SMV reader (ADSC), cubic F4132"}},
|
||||
{"id": "6p8p", "input": "6p8p/14_12_1_0001.cbf", "ref": {"sg": "P 4", "sgno": 75, "cell": [97.525, 97.525, 60.134, 90.0, 90.0, 90.0], "dmin": 1.635}, "tags": ["cbf", "tetragonal"]},
|
||||
{"id": "6pb3", "input": "6pb3/14_16_1_000001.cbf", "ref": {"sg": "P 6", "sgno": 168, "cell": [100.436, 100.436, 48.86, 90.0, 90.0, 120.0], "dmin": 2.048}, "tags": ["cbf", "hexagonal"]},
|
||||
{"id": "6pxb", "input": "6pxb/TJB1_3_rachel_1_0001.cbf", "ref": {"sg": "P 32", "sgno": 145, "cell": [64.021, 64.021, 119.447, 90.0, 90.0, 120.0], "dmin": 1.747}, "tags": ["cbf", "trigonal"]},
|
||||
{"id": "6pxc", "input": "6pxc/TJB6_1_Rachel_1_0001.cbf", "ref": {"sg": "I 2 2 2", "sgno": 23, "cell": [44.19, 64.827, 87.239, 90.0, 90.0, 90.0], "dmin": 1.6}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "6r72", "input": "6r72/V-CK63-8-ld_1_master.h5", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [117.805, 110.754, 155.615, 90.0, 93.23, 90.0], "dmin": 3.95}, "pinned": true, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "6rlr", "input": "6rlr/_b4_1_master.h5", "ref": {"sg": "P 1", "sgno": 1, "cell": [39.986, 39.998, 63.643, 80.39, 76.29, 68.15], "dmin": 2.0}, "tags": ["h5", "triclinic", "twin"]},
|
||||
{"id": "6toc", "input": "6toc/zenodo/dts_1_00001.cbf", "ref": {"sg": "P 42", "sgno": 77, "cell": [31.523, 31.523, 81.599, 90.0, 90.0, 90.0], "dmin": 1.853}, "tags": ["cbf", "tetragonal", "twin"]},
|
||||
{"id": "6ttn", "input": "6ttn/6ttn/data/H6H_33_01_hyo_full_1_0001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [39.894, 79.841, 104.701, 90.0, 90.0, 90.0], "dmin": 1.12}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "6u7g", "input": "6u7g/6u7g/data/SNB02_11_8_1_master.h5", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [99.555, 98.682, 147.525, 90.0, 104.6, 90.0], "dmin": 2.35}, "pinned": true, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "6ukf", "input": "6ukf/6ukf/data/XDC-7_Pn6_000001.cbf", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [61.018, 37.317, 69.027, 90.0, 109.768, 90.0], "dmin": 1.0}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "6vww", "input": "6vww/IDP51000_6vww/data/m4H11g_00001.cbf", "ref": {"sg": "P 63", "sgno": 173, "cell": [150.539, 150.539, 111.31, 90.0, 90.0, 120.0], "dmin": 2.2}, "tags": ["cbf", "hexagonal", "twin"], "tiers": {"smoke": "merohedral twin, P63"}},
|
||||
{"id": "6w4h", "input": "6w4h/IDP51000_6W4H/data/idp51000-201-a_1_2_3.001", "ref": {"sg": "P 31 2 1", "sgno": 152, "cell": [167.74, 167.74, 51.942, 90.0, 90.0, 120.0], "dmin": 1.8}, "tags": ["marCCD", "trigonal"]},
|
||||
{"id": "6wzo", "input": "6wzo/9_1_1_000001.cbf", "ref": {"sg": "P 1", "sgno": 1, "cell": [43.718, 50.061, 69.337, 106.499, 90.094, 97.145], "dmin": 1.42}, "tags": ["cbf", "triclinic"]},
|
||||
{"id": "6yqf", "input": "6yqf/6yqf/data/SYCE2TEX12-8_1_0001.cbf", "ref": {"sg": "P 21 21 2", "sgno": 18, "cell": [42.67, 59.68, 156.49, 90.0, 90.0, 90.0], "dmin": 3.327}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "6z8o", "input": "6z8o/HASE01-gaKr1_w1_1_1_master.h5", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [63.716, 97.022, 121.318, 90.0, 104.656, 90.0], "dmin": 2.2}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "6ze4", "input": "6ze4/806/o8_1_0001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [93.56, 109.88, 116.11, 90.0, 90.0, 90.0], "dmin": 1.6}, "tags": ["cbf", "orthorhombic"], "tiers": {"smoke": "known hard: P212121 under-called as P21 since rc168"}},
|
||||
{"id": "7arr", "input": "7arr/PF4N_04D_w1_1_00001.cbf", "ref": {"sg": "P 1", "sgno": 1, "cell": [30.937, 32.068, 43.087, 114.16, 91.88, 109.86], "dmin": 1.1}, "tags": ["cbf", "triclinic"]},
|
||||
{"id": "7atg", "input": "7atg/7atg/data/put-2-2.0s_5_00001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [17.97, 31.02, 43.86, 90.0, 90.0, 90.0], "dmin": 0.6}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "7bgt", "input": "7bgt/MPMV2243_3_001.img", "ref": {"sg": "P 1", "sgno": 1, "cell": [29.296, 67.618, 69.716, 76.845, 83.875, 83.645], "dmin": 1.93}, "tags": ["marCCD", "triclinic"]},
|
||||
{"id": "7brr", "input": "7brr/7brr/data/3C_2_1_master.h5", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [55.45, 99.02, 59.583, 90.0, 108.54, 90.0], "dmin": 1.35}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "7dkp", "input": "7dkp/7dkp/data/LOX1-sree-p1a9-1p2_w1_1_1_master.h5", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [49.84, 169.451, 49.842, 90.0, 93.51, 90.0], "dmin": 1.45}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "7k1l", "input": "7k1l/IDP52015_7k1l/data/cs1C2eg_00001.cbf", "ref": {"sg": "P 63", "sgno": 173, "cell": [150.83, 150.83, 110.73, 90.0, 90.0, 120.0], "dmin": 2.25}, "tags": ["cbf", "hexagonal"], "tiers": {"smoke": "known hard: screw axis (P6 vs P63)"}},
|
||||
{"id": "7kcn", "input": "7kcn/7kcn/data/ttr_hiuase_1_D1_33_00001.cbf", "ref": {"sg": "P 41 2 2", "sgno": 91, "cell": [67.03, 67.03, 116.89, 90.0, 90.0, 90.0], "dmin": 1.46}, "tags": ["cbf", "tetragonal"]},
|
||||
{"id": "7l6j", "input": "7l6j/7l6j/data/idp97824-102sm-a_1_1_7.001", "ref": {"sg": "I 41 3 2", "sgno": 214, "cell": [171.693, 171.693, 171.693, 90.0, 90.0, 90.0], "dmin": 1.78}, "tags": ["marCCD", "cubic"]},
|
||||
{"id": "7l84", "input": "7l84/301_helical_1_0001.cbf", "ref": {"sg": "P 43 21 2", "sgno": 96, "cell": [79.344, 79.344, 37.81, 90.0, 90.0, 90.0], "dmin": 1.7}, "tags": ["cbf", "tetragonal"]},
|
||||
{"id": "7mzt", "input": "7mzt/7mzt/data/BVG2_Pn3_000001.cbf", "ref": {"sg": "P 21 21 2", "sgno": 18, "cell": [113.619, 96.989, 108.33, 90.0, 90.0, 90.0], "dmin": 4.07}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "7n0i", "input": "7n0i/8_2_1_00001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [75.846, 131.557, 140.048, 90.0, 90.0, 90.0], "dmin": 2.2}, "tags": ["cbf", "orthorhombic", "tncs"]},
|
||||
{"id": "7n2s", "input": "7n2s/1449-4_1_00001.cbf", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [83.211, 52.789, 106.291, 90.0, 98.278, 90.0], "dmin": 2.37}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "7orr", "input": "7orr/7orr/data/Nsp10-VT00022_1_master.h5", "ref": {"sg": "I 21 3", "sgno": 199, "cell": [105.88, 105.88, 105.88, 90.0, 90.0, 90.0], "dmin": 1.79}, "tags": ["h5", "cubic"]},
|
||||
{"id": "7os3", "input": "7os3/pos2_1/pos2_1_00001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [78.15, 91.05, 105.84, 90.0, 90.0, 90.0], "dmin": 2.177}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "7ph1", "input": "7ph1/7ph1/data/9_1_0001.cbf", "ref": {"sg": "I 2 2 2", "sgno": 23, "cell": [74.975, 81.289, 124.248, 90.0, 90.0, 90.0], "dmin": 1.18}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "7pq7", "input": "7pq7/7pq7/data/1436_p52_A3_1_1_00001.cbf", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [120.88, 51.73, 75.54, 90.0, 125.14, 90.0], "dmin": 1.55}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "7qij", "input": "7qij/907/dg046_2_data_1_00001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [143.46, 324.92, 369.38, 90.0, 90.0, 90.0], "dmin": 4.1}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "7qis", "input": "7qis/7qis/data/Chymo_Dfe_x1_2_0001.cbf", "ref": {"sg": "P 61", "sgno": 169, "cell": [100.269, 100.269, 206.215, 90.0, 90.0, 120.0], "dmin": 1.83}, "tags": ["cbf", "hexagonal"]},
|
||||
{"id": "7ris", "input": "7ris/7ris/data/GLVase_Ca_we21108b7b_1_2_2.001_master.h5", "ref": {"sg": "P 32 2 1", "sgno": 154, "cell": [44.54, 44.54, 189.95, 90.0, 90.0, 120.0], "dmin": 1.72}, "pinned": true, "tags": ["h5", "trigonal"]},
|
||||
{"id": "7rji", "input": "7rji/7rji/data/DD3_00000.cbf", "ref": {"sg": "H 3 2", "sgno": 155, "cell": [83.036, 83.036, 124.83, 90.0, 90.0, 120.0], "dmin": 1.71}, "tags": ["cbf", "trigonal"]},
|
||||
{"id": "7t5t", "input": "7t5t/A1_2_00001.cbf", "ref": {"sg": "P 42 21 2", "sgno": 94, "cell": [95.309, 95.309, 104.915, 90.0, 90.0, 90.0], "dmin": 1.35}, "pinned": true, "tags": ["cbf", "tetragonal"]},
|
||||
{"id": "7tcd", "input": "7tcd/7tcd/data/col_1_master.h5", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [138.504, 47.902, 78.068, 90.0, 107.56, 90.0], "dmin": 1.7}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "7yzx", "input": "7yzx/7yzx/data/783-1_1_0001.cbf", "ref": {"sg": "P 63 2 2", "sgno": 182, "cell": [169.42, 169.42, 141.83, 90.0, 90.0, 120.0], "dmin": 1.9}, "tags": ["cbf", "hexagonal"]},
|
||||
{"id": "8a1a", "input": "8a1a/L1F11v1_8a1a/data/L1-F11-v1_9ly1_D06_01_2_master.h5", "ref": {"sg": "P 65", "sgno": 170, "cell": [191.866, 191.866, 122.409, 90.0, 90.0, 120.0], "dmin": 2.05}, "tags": ["h5", "hexagonal"]},
|
||||
{"id": "8agq", "input": "8agq/8agq_zip_8AGQ/data/coll_1_00001.cbf", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [89.948, 55.447, 54.787, 90.0, 113.55, 90.0], "dmin": 1.093}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "8dqb", "input": "8dqb/KloxA_17380_a_B1-Apo_I23_Form_8dqb/data/PSL-1405_653_master.h5", "ref": {"sg": "I 2 3", "sgno": 197, "cell": [164.124, 164.124, 164.124, 90.0, 90.0, 90.0], "dmin": 2.5}, "tags": ["h5", "cubic"]},
|
||||
{"id": "8dyz", "input": "8dyz/lys_1_2_0001.cbf", "ref": {"sg": "P 43 21 2", "sgno": 96, "cell": [79.632, 79.632, 38.296, 90.0, 90.0, 90.0], "dmin": 1.272}, "tags": ["cbf", "tetragonal", "diffuse"]},
|
||||
{"id": "8dz7", "input": "8dz7/lys_rt_2_38_0001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [30.486, 56.401, 73.852, 90.0, 90.0, 90.0], "dmin": 1.34}, "tags": ["cbf", "orthorhombic", "diffuse"]},
|
||||
{"id": "8egn", "input": "8egn/PsaeA_00137_b_B5-az13643701_8egn/data/FKC8_4_0001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [71.69, 75.16, 109.79, 90.0, 90.0, 90.0], "dmin": 1.95}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "8iya", "input": "8iya/8iya/data/11_1_master.h5", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [102.747, 50.077, 109.248, 90.0, 91.754, 90.0], "dmin": 2.43}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "8k1g", "input": "8k1g/8k1g/data/KP02_0001.cbf", "ref": {"sg": "I 4 2 2", "sgno": 97, "cell": [182.01, 182.01, 80.71, 90.0, 90.0, 90.0], "dmin": 2.09}, "tags": ["cbf", "tetragonal"]},
|
||||
{"id": "8oic", "input": "8oic/8oic/data/_b20_1_master.h5", "ref": {"sg": "P 1", "sgno": 1, "cell": [73.055, 94.665, 120.589, 105.081, 89.959, 93.826], "dmin": 2.8}, "tags": ["h5", "triclinic"]},
|
||||
{"id": "8owm", "input": "8owm/p13x4_1_00001.cbf", "ref": {"sg": "P 1", "sgno": 1, "cell": [95.544, 95.629, 95.841, 90.416, 93.59, 117.775], "dmin": 1.7}, "tags": ["cbf", "triclinic"]},
|
||||
{"id": "8pqd", "input": "8pqd/8pqd/data/11AT05_w1_1_1_master.h5", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [59.39, 59.37, 192.89, 90.0, 90.0, 90.0], "dmin": 1.5}, "tags": ["h5", "orthorhombic"]},
|
||||
{"id": "8qaw", "input": "8qaw/p18x3/p18x3_1_00001.cbf.gz", "ref": {"sg": "H 3", "sgno": 146, "cell": [137.68, 137.68, 265.907, 90.0, 90.0, 120.0], "dmin": 1.55}, "tags": ["cbf", "trigonal", "long-axis"]},
|
||||
{"id": "8qj5", "input": "8qj5/8qj5/data/Lcsbc3_1_2_00001.cbf", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [57.61, 100.63, 77.93, 90.0, 96.06, 90.0], "dmin": 1.63}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "8qq7", "input": "8qq7/cbf/SpNox-Sp49-B3-B_088_1_0001.cbf", "ref": {"sg": "P 64 2 2", "sgno": 181, "cell": [145.967, 145.967, 153.619, 90.0, 90.0, 120.0], "dmin": 3.62}, "tags": ["cbf", "hexagonal"]},
|
||||
{"id": "8r5r", "input": "8r5r/8r5r/data/D5-33805_1_1_master.h5", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [91.74, 132.91, 137.5, 90.0, 90.0, 90.0], "dmin": 3.08}, "tags": ["h5", "orthorhombic"]},
|
||||
{"id": "8rud", "input": "8rud/p08x03_1_00001.cbf", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [78.066, 91.386, 114.547, 90.0, 96.9, 90.0], "dmin": 2.1}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "8s38", "input": "8s38/MX733_2_00001.cbf", "ref": {"sg": "I 21 21 21", "sgno": 24, "cell": [95.368, 163.1, 219.023, 90.0, 90.0, 90.0], "dmin": 1.89}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "8sa8", "input": "8sa8/KlaeA_00906_a_B1-PLP_8sa8/data/PSL-0412_255_master.h5", "ref": {"sg": "I 1 2 1", "sgno": 5, "cell": [87.887, 131.544, 165.363, 90.0, 104.48, 90.0], "dmin": 1.3}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "8sqo", "input": "8sqo/BrabA_00028_a_A1-Mg_8sqo/data/PSL-1801_942_master.h5", "ref": {"sg": "P 4 3 2", "sgno": 207, "cell": [112.91, 112.91, 112.91, 90.0, 90.0, 90.0], "dmin": 1.55}, "tags": ["h5", "cubic"]},
|
||||
{"id": "8sqq", "input": "8sqq/BrabA_00028_a_A1-Apo_8sqq/data/PSL-1810_965_master.h5", "ref": {"sg": "F 4 3 2", "sgno": 209, "cell": [171.52, 171.52, 171.52, 90.0, 90.0, 90.0], "dmin": 2.25}, "tags": ["h5", "cubic"]},
|
||||
{"id": "8sqt", "input": "8sqt/BrabA_00028_a_A1-Fe_8sqt/data/PSL-1812_971_master.h5", "ref": {"sg": "F 4 3 2", "sgno": 209, "cell": [170.72, 170.72, 170.72, 90.0, 90.0, 90.0], "dmin": 2.2}, "tags": ["h5", "cubic"]},
|
||||
{"id": "8t7r", "input": "8t7r/8t7r/data/GREEN-04_Pn5_000001.cbf", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [357.076, 259.582, 255.361, 90.0, 133.13, 90.0], "dmin": 3.84}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "8tha", "input": "8tha/1TEL_double_helix_8tha/data/A3_1_00001.cbf", "ref": {"sg": "P 64", "sgno": 172, "cell": [69.16, 69.16, 29.13, 90.0, 90.0, 120.0], "dmin": 1.68}, "tags": ["cbf", "hexagonal"]},
|
||||
{"id": "8tyy", "input": "8tyy/CPS_2_1_000001.cbf", "ref": {"sg": "F 4 3 2", "sgno": 209, "cell": [214.891, 214.891, 214.891, 90.0, 90.0, 90.0], "dmin": 1.68}, "tags": ["cbf", "cubic"]},
|
||||
{"id": "8u0i", "input": "8u0i/8u0i/data/dI-2-05a_4_00001.cbf", "ref": {"sg": "P 43 21 2", "sgno": 96, "cell": [50.317, 50.317, 90.596, 90.0, 90.0, 90.0], "dmin": 1.541}, "tags": ["cbf", "tetragonal"]},
|
||||
{"id": "8v4o", "input": "8v4o/CaalA_00629_a_FS11-AMP_8v4o/data/PSL-1602_2180_master.h5", "ref": {"sg": "P 61 2 2", "sgno": 178, "cell": [139.46, 139.46, 544.99, 90.0, 90.0, 120.0], "dmin": 2.7}, "tags": ["h5", "hexagonal"]},
|
||||
{"id": "8xbp", "input": "8xbp/8xbp/data/puck2872_3_1_master.h5", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [148.29, 50.78, 60.21, 90.0, 92.33, 90.0], "dmin": 1.99}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "8xte", "input": "8xte/1101/G2_1_00001.cbf", "ref": {"sg": "P 32 2 1", "sgno": 154, "cell": [208.77, 208.77, 67.22, 90.0, 90.0, 120.0], "dmin": 1.99}, "tags": ["cbf", "trigonal"]},
|
||||
{"id": "8xtf", "input": "8xtf/1102/puck03_12_1_master.h5", "ref": {"sg": "H 3 2", "sgno": 155, "cell": [211.75, 211.75, 67.42, 90.0, 90.0, 120.0], "dmin": 2.13}, "tags": ["h5", "trigonal"]},
|
||||
{"id": "8xtg", "input": "8xtg/1100/mpag15_1_00001.cbf", "ref": {"sg": "P 32 2 1", "sgno": 154, "cell": [199.54, 199.54, 67.15, 90.0, 90.0, 120.0], "dmin": 2.0}, "tags": ["cbf", "trigonal"]},
|
||||
{"id": "8y74", "input": "8y74/1_1/1_1_master.h5", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [125.782, 76.553, 87.143, 90.0, 92.449, 90.0], "dmin": 1.9}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "8ys9", "input": "8ys9/8YS9_xray-data_8ys9/data/SDS_09_18112_master.h5", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [71.04, 77.68, 83.16, 90.0, 90.0, 90.0], "dmin": 1.46}, "tags": ["h5", "orthorhombic"]},
|
||||
{"id": "9b22", "input": "9b22/KlpnC_20447_a_B1-AR6-AMP_9b22/data/PSL-1011_1580_master.h5", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [39.826, 92.659, 57.716, 90.0, 91.68, 90.0], "dmin": 1.3}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "9bn8", "input": "9bn8/EscoA_17938_a_AE1-UMA-A1AQS_9bn8/data/PSL-0401_8296_master.h5", "ref": {"sg": "P 41", "sgno": 76, "cell": [65.488, 65.488, 134.764, 90.0, 90.0, 90.0], "dmin": 1.35}, "tags": ["h5", "tetragonal"]},
|
||||
{"id": "9c18", "input": "9c18/BLVRB-2_5217_master.h5", "ref": {"sg": "P 1", "sgno": 1, "cell": [41.94, 41.977, 60.212, 84.11, 87.24, 63.66], "dmin": 1.9}, "tags": ["h5", "triclinic"]},
|
||||
{"id": "9chw", "input": "9chw/D1.001", "ref": {"sg": "P 61", "sgno": 169, "cell": [98.686, 98.686, 82.062, 90.0, 90.0, 120.0], "dmin": 2.16}, "tags": ["marCCD", "hexagonal"]},
|
||||
{"id": "9crw", "input": "9crw/data_9crw/data/Kip3-475-22-2_master.h5", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [83.96, 104.57, 118.78, 90.0, 93.37, 90.0], "dmin": 2.49}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "9e2t", "input": "9e2t/ga026f4b-1_5_00001.cbf", "ref": {"sg": "P 1", "sgno": 1, "cell": [75.499, 78.082, 101.219, 94.63, 103.39, 114.55], "dmin": 2.28}, "tags": ["cbf", "triclinic"]},
|
||||
{"id": "9ea5", "input": "9ea5/K4_1_00001.cbf", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [65.868, 73.064, 98.399, 90.0, 108.725, 90.0], "dmin": 2.0}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "9fcg", "input": "9fcg/IBCH-15-p02x06_2_00001.cbf.gz", "ref": {"sg": "P 4", "sgno": 75, "cell": [87.79, 87.79, 35.633, 90.0, 90.0, 90.0], "dmin": 1.54}, "tags": ["cbf", "tetragonal"], "tiers": {"smoke": "cbf.gz reader, pure 4-fold"}},
|
||||
{"id": "9fhc", "input": "9fhc/te/te312_2_001.img", "ref": {"sg": "I 2 3", "sgno": 197, "cell": [227.46, 227.46, 227.46, 90.0, 90.0, 90.0], "dmin": 2.2}, "tags": ["marCCD", "cubic"]},
|
||||
{"id": "9gdj", "input": "9gdj/slh34_w1_1_1_master.h5", "ref": {"sg": "P 41 21 2", "sgno": 92, "cell": [123.92, 123.92, 126.44, 90.0, 90.0, 90.0], "dmin": 1.47}, "tags": ["h5", "tetragonal", "cdte"]},
|
||||
{"id": "9gjx", "input": "9gjx/BlNTR_9gjx/data/SC6_1_master.h5", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [76.776, 115.807, 103.807, 90.0, 110.29, 90.0], "dmin": 2.4}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "9gqg", "input": "9gqg/MX-2555/FKBP51-MSP500-A-5522-11_1_3094759/FKBP51-MSP500-A-5522-11_1_1_master.h5", "ref": {"sg": "P 32 2 1", "sgno": 154, "cell": [48.157, 48.157, 188.036, 90.0, 90.0, 120.0], "dmin": 2.0}, "tags": ["h5", "trigonal"]},
|
||||
{"id": "9hnc", "input": "9hnc/asp_7_00001.cbf", "ref": {"sg": "P 1 2 1", "sgno": 3, "cell": [123.764, 123.645, 187.679, 90.0, 90.063, 90.0], "dmin": 1.879}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "9hs7", "input": "9hs7/9hs7/data/B28X1_1_0001.cbf", "ref": {"sg": "P 65", "sgno": 170, "cell": [65.44, 65.44, 88.77, 90.0, 90.0, 120.0], "dmin": 1.698}, "tags": ["cbf", "hexagonal"]},
|
||||
{"id": "9i0a", "input": "9i0a/carm1END468_9i0a_tar_9I0A/data/X151B1_1_master.h5", "ref": {"sg": "P 21 21 2", "sgno": 18, "cell": [75.237, 98.676, 208.551, 90.0, 90.0, 90.0], "dmin": 2.22}, "tags": ["h5", "orthorhombic"]},
|
||||
{"id": "9i80", "input": "9i80/20230422-PX1-LecA_RO5-5315-1/LecA-RO5_1_master.h5", "ref": {"sg": "P 41", "sgno": 76, "cell": [81.214, 81.214, 165.029, 90.0, 90.0, 90.0], "dmin": 1.95}, "tags": ["h5", "tetragonal", "twin"]},
|
||||
{"id": "9ig7", "input": "9ig7/9IG7/data/KOD-P596-G2_3_00001.cbf", "ref": {"sg": "P 21 21 2", "sgno": 18, "cell": [111.472, 153.471, 69.03, 90.0, 90.0, 90.0], "dmin": 2.6}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "9ih9", "input": "9ih9/run_01_05_datacollection_9IH9/data/design-Xinjian-P1Thro-4_1_5_1_master.h5", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [78.758, 133.933, 82.32, 90.0, 101.356, 90.0], "dmin": 1.7}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "9jzo", "input": "9jzo/JS_35P2_C3-002_diffraction_files_9JZO/data/JS_35P2_C3-002_0001.cbf", "ref": {"sg": "P 1", "sgno": 1, "cell": [41.63, 43.1, 54.2, 112.97, 90.11, 118.18], "dmin": 1.4}, "tags": ["cbf", "triclinic"], "tiers": {"smoke": "triclinic P1"}},
|
||||
{"id": "9khr", "input": "9khr/PfPlrx-DTT-9KHR/data-PfPlrx-DTT-9KHR/ods1467-lp13D1-01001.mccd", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [48.704, 50.31, 78.018, 90.0, 90.0, 90.0], "dmin": 2.0}, "tags": ["marCCD", "orthorhombic"], "tiers": {"smoke": "marCCD reader (.mccd), fast"}},
|
||||
{"id": "9mh4", "input": "9mh4/KlaeA_00150_a_B1_9mh4/data/PSL-0501_2719_master.h5", "ref": {"sg": "P 21 3", "sgno": 198, "cell": [138.65, 138.65, 138.65, 90.0, 90.0, 90.0], "dmin": 3.05}, "tags": ["h5", "cubic"]},
|
||||
{"id": "9min", "input": "9min/1151/A8_31_00001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [95.45, 98.54, 155.74, 90.0, 90.0, 90.0], "dmin": 2.05}, "tags": ["cbf", "orthorhombic"], "tiers": {"smoke": "known hard: halved axis"}},
|
||||
{"id": "9o0h", "input": "9o0h/3TEL-GG-TNK1-UBA_9o0h/data/E1_1_00001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [55.15, 65.54, 112.9, 90.0, 90.0, 90.0], "dmin": 2.24}, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "9p7q", "input": "9p7q/9p7q_9P7Q/data/SAMDC_MTA_273_40_00001.cbf", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [96.978, 45.018, 72.061, 90.0, 105.109, 90.0], "dmin": 2.21}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "9pbb", "input": "9pbb/9pbb_9PBB/data/SAMDC_MTA_293_30_00001.cbf", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [97.418, 45.876, 72.248, 90.0, 104.971, 90.0], "dmin": 2.17}, "pinned": true, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "9q41", "input": "9q41/apoSPDS_1_master.h5", "ref": {"sg": "C 2 2 21", "sgno": 20, "cell": [118.568, 133.704, 82.369, 90.0, 90.0, 90.0], "dmin": 1.95}, "tags": ["h5", "orthorhombic"]},
|
||||
{"id": "9q66", "input": "9q66/PREP_JP417_D8a_12694_master.h5", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [105.853, 67.294, 158.019, 90.0, 99.109, 90.0], "dmin": 2.01}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "9qw8", "input": "9qw8/MX-2555/run_03_datacollection [2023-09-08 22:19:51]/FKBP12BRD4-5519-6-BD1-PPU688_w1_1_1_master.h5", "ref": {"sg": "P 1", "sgno": 1, "cell": [35.648, 35.65, 100.919, 86.458, 84.218, 72.475], "dmin": 1.8}, "tags": ["h5", "triclinic"]},
|
||||
{"id": "9rci", "input": "9rci/FEN1-CD029134_A09-3_AD049A-10_1_master.h5", "ref": {"sg": "P 1", "sgno": 1, "cell": [35.869, 39.297, 100.916, 98.3, 90.32, 90.09], "dmin": 1.662}, "tags": ["h5", "triclinic", "tncs"]},
|
||||
{"id": "9rcs", "input": "9rcs/760_B7_x1_1_master.h5", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [70.01, 78.8, 82.34, 90.0, 88.64, 90.0], "dmin": 3.012}, "tags": ["h5", "monoclinic", "cdte"]},
|
||||
{"id": "9rp9", "input": "9rp9/data_9RP9/data/TAR148-TAR148_X2_1_master.h5", "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [73.48, 59.78, 91.67, 90.0, 100.85, 90.0], "dmin": 2.1}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "9sl0", "input": "9sl0/wt-1_9sl0/data/design-Yan-HLA-WT-1_1_5_1_master.h5", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [60.181, 80.201, 111.589, 90.0, 90.0, 90.0], "dmin": 1.6}, "tags": ["h5", "orthorhombic"]},
|
||||
{"id": "9t6s", "input": "9t6s/CadC-AF6586_1_5_1_master.h5", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [63.004, 64.574, 102.705, 90.0, 90.0, 90.0], "dmin": 2.0}, "tags": ["h5", "orthorhombic", "tncs"]},
|
||||
{"id": "9upt", "input": "9upt/Raw data 1A/C12_7_00001.img", "ref": {"sg": "P 6", "sgno": 168, "cell": [158.27, 158.27, 53.962, 90.0, 90.0, 120.0], "dmin": 2.369}, "tags": ["SMV", "hexagonal"]},
|
||||
{"id": "9vyb", "input": "9vyb/Antitoxin Phd_9vyb/data/G7-5_4187_master.h5", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [44.37, 47.76, 48.35, 90.0, 90.0, 90.0], "dmin": 2.12}, "tags": ["h5", "orthorhombic"]},
|
||||
{"id": "9w3y", "input": "9w3y/9w3y/data/collect_01_00001_master.h5", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [60.75, 69.98, 94.24, 90.0, 90.0, 90.0], "dmin": 1.5}, "tags": ["h5", "orthorhombic"]},
|
||||
{"id": "9yl4", "input": "9yl4/data_MJ4720/UWO0004_06_0001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [95.837, 111.339, 402.967, 90.0, 90.0, 90.0], "dmin": 3.7}, "pinned": true, "tags": ["cbf", "orthorhombic"]},
|
||||
{"id": "9yzk", "input": "9yzk/CC1_21_TGGGAAGATACTGGATGAGGT_empty_9yzk/data/run_29_00001.cbf", "ref": {"sg": "I 1 2 1", "sgno": 5, "cell": [75.836, 163.022, 192.278, 90.0, 98.614, 90.0], "dmin": 5.1}, "tags": ["cbf", "monoclinic"], "tiers": {"smoke": "I2/C2 setting choice at low resolution"}},
|
||||
{"id": "9z44", "input": "9z44/CC1_10_CCCGGCCGG_Cclamp_9z44/data/run_41_00001.cbf", "ref": {"sg": "I 1 2 1", "sgno": 5, "cell": [73.491, 127.653, 141.183, 90.0, 91.984, 90.0], "dmin": 7.2}, "tags": ["cbf", "monoclinic"]},
|
||||
{"id": "9z72", "input": "9z72/A7_1_00001.cbf", "ref": {"sg": "P 31 2 1", "sgno": 152, "cell": [59.173, 59.173, 426.203, 90.0, 90.0, 120.0], "dmin": 2.38}, "tags": ["cbf", "trigonal", "long-axis"], "tiers": {"smoke": "long axis c=426 A, at the FFT ceiling"}},
|
||||
{"id": "9zlo", "input": "9zlo/ASP0887_09_0114_master.h5", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [38.39, 89.98, 106.97, 90.0, 90.0, 90.0], "dmin": 2.0}, "tags": ["h5", "orthorhombic"]},
|
||||
{"id": "9zm0", "input": "9zm0/Atg23ANNS_9zm0_9ZM0/data/SeAtg23_B_10_2115_master.h5", "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [50.409, 30.101, 91.244, 90.0, 97.146, 90.0], "dmin": 2.1}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "9zmu", "input": "9zmu/BrmeA_18154_a_B2-Apo_9zmu/data/PSL-0613_7518_master.h5", "ref": {"sg": "P 65 2 2", "sgno": 179, "cell": [47.81, 47.81, 492.58, 90.0, 90.0, 120.0], "dmin": 1.98}, "tags": ["h5", "hexagonal"]},
|
||||
{"id": "cuhf2", "input": "cuhf2/03_CuHF2pyz2PF6b_P_O/CuHF2pyz2PF6b_P_O_01.nxs", "pinned": true, "tags": ["nxs", "small-molecule"]},
|
||||
{"id": "cytidine", "input": "cytidine/20151020-Cytidine-2th-30/fixed-omega--180-phi-scan01_00001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [13.98, 14.788, 5.119, 90.0, 90.0, 90.0], "dmin": null}, "tags": ["cbf", "orthorhombic", "small-molecule"]},
|
||||
{"id": "dnba", "input": "dnba/35dnba_30K_2_04_00001.cbf", "ref": {"sg": "C 1 2/c 1", "sgno": 15, "cell": [20.2635, 8.7575, 9.6697, 90.0, 109.941, 90.0], "dmin": 0.48}, "tags": ["cbf", "monoclinic", "small-molecule"]},
|
||||
{"id": "lalanine", "input": "lalanine/pgw240050_01_00001.cbf", "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [5.7952, 5.933, 12.362, 90.0, 90.0, 90.0], "dmin": null}, "tags": ["cbf", "orthorhombic", "small-molecule"], "tiers": {"smoke": "small molecule, miniCBF"}},
|
||||
{"id": "metformin", "input": "metformin/013_Mmetformin_01_master.h5", "ref": {"sg": "P 1 21/c 1", "sgno": 14, "cell": [7.9104, 13.8794, 7.931, 90.0, 114.606, 90.0], "dmin": 0.45}, "tags": ["h5", "monoclinic", "small-molecule"], "tiers": {"smoke": "small molecule, Diamond I19 NXmx master"}},
|
||||
{"id": "nidppe", "input": "nidppe/001_NiDppeCl2_01_master.h5", "ref": {"sg": "P 1 21/c 1", "sgno": 14, "cell": [11.2779, 13.3386, 15.8739, 90.0, 98.7953, 90.0], "dmin": 0.77}, "tags": ["h5", "monoclinic", "small-molecule"]}
|
||||
]}
|
||||
@@ -0,0 +1,107 @@
|
||||
"""One document, written twice: Markdown and a self-contained HTML page.
|
||||
|
||||
The report is built as a list of blocks (heading, paragraph, bullet list, table) and each writer
|
||||
turns the same list into its format, so the two can never say different things.
|
||||
"""
|
||||
import html
|
||||
|
||||
CSS = """
|
||||
:root { --bg:#fff; --fg:#1d1d1f; --mute:#6e6e73; --line:#d9d9de; --head:#f3f3f5;
|
||||
--pass:#1a7f37; --fail:#c62828; --warn:#9a6700; --passbg:#e8f5ec; --failbg:#fdecea; }
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root { --bg:#161618; --fg:#e8e8ea; --mute:#9a9aa0; --line:#3a3a3e; --head:#232326;
|
||||
--pass:#5cc97a; --fail:#ff7b72; --warn:#e3b341; --passbg:#16301f; --failbg:#3a1a1a; }
|
||||
}
|
||||
body { background:var(--bg); color:var(--fg); margin:0 auto; max-width:1400px; padding:16px;
|
||||
font:14px/1.45 system-ui, -apple-system, "Segoe UI", sans-serif; }
|
||||
h1 { font-size:22px; margin:8px 0 4px; } h2 { font-size:18px; margin:28px 0 8px; }
|
||||
h3 { font-size:15px; margin:20px 0 6px; }
|
||||
p, li { max-width:900px; } .mute { color:var(--mute); }
|
||||
.wrap { overflow-x:auto; margin:8px 0 16px; }
|
||||
table { border-collapse:collapse; font-variant-numeric:tabular-nums; }
|
||||
th, td { border-bottom:1px solid var(--line); padding:4px 8px; text-align:left; white-space:nowrap; }
|
||||
th { background:var(--head); position:sticky; top:0; }
|
||||
td.num, th.num { text-align:right; }
|
||||
td.pass { color:var(--pass); font-weight:600; } td.fail { color:var(--fail); font-weight:600; }
|
||||
td.warn { color:var(--warn); font-weight:600; }
|
||||
tr.fail td { background:var(--failbg); }
|
||||
code { font:12px ui-monospace, Menlo, monospace; }
|
||||
"""
|
||||
|
||||
|
||||
class Doc:
|
||||
def __init__(self, title):
|
||||
self.title = title
|
||||
self.blocks = []
|
||||
|
||||
def h(self, level, text):
|
||||
self.blocks.append(("h", level, text))
|
||||
|
||||
def p(self, text):
|
||||
self.blocks.append(("p", text))
|
||||
|
||||
def ul(self, items):
|
||||
self.blocks.append(("ul", list(items)))
|
||||
|
||||
def table(self, head, rows, num=(), row_class=None, cell_class=None):
|
||||
"""head: column titles; rows: lists of str; num: indices of right-aligned columns;
|
||||
row_class(row) and cell_class(i, value) give optional CSS classes (HTML only)."""
|
||||
self.blocks.append(("table", head, rows, set(num), row_class, cell_class))
|
||||
|
||||
def markdown(self):
|
||||
out = [f"# {self.title}", ""]
|
||||
for b in self.blocks:
|
||||
if b[0] == "h":
|
||||
out += ["#" * b[1] + " " + b[2], ""]
|
||||
elif b[0] == "p":
|
||||
out += [b[1], ""]
|
||||
elif b[0] == "ul":
|
||||
out += ["- " + x for x in b[1]] + [""]
|
||||
else:
|
||||
_, head, rows, num, _, _ = b
|
||||
if not rows:
|
||||
out += ["(none)", ""]
|
||||
continue
|
||||
out.append("| " + " | ".join(head) + " |")
|
||||
out.append("|" + "|".join("--:" if i in num else ":--" for i in range(len(head))) + "|")
|
||||
for r in rows:
|
||||
out.append("| " + " | ".join(str(x).replace("|", "/") for x in r) + " |")
|
||||
out.append("")
|
||||
return "\n".join(out)
|
||||
|
||||
def html(self):
|
||||
e = html.escape
|
||||
out = ["<!DOCTYPE html>", '<html lang="en"><head><meta charset="utf-8">',
|
||||
'<meta name="viewport" content="width=device-width, initial-scale=1">',
|
||||
f"<title>{e(self.title)}</title><style>{CSS}</style></head><body>",
|
||||
f"<h1>{e(self.title)}</h1>"]
|
||||
for b in self.blocks:
|
||||
if b[0] == "h":
|
||||
out.append(f"<h{b[1]}>{e(b[2])}</h{b[1]}>")
|
||||
elif b[0] == "p":
|
||||
out.append(f"<p>{e(b[1])}</p>")
|
||||
elif b[0] == "ul":
|
||||
out.append("<ul>" + "".join(f"<li>{e(x)}</li>" for x in b[1]) + "</ul>")
|
||||
else:
|
||||
_, head, rows, num, row_class, cell_class = b
|
||||
if not rows:
|
||||
out.append('<p class="mute">(none)</p>')
|
||||
continue
|
||||
t = ['<div class="wrap"><table><thead><tr>']
|
||||
t += [f'<th class="num">{e(h)}</th>' if i in num else f"<th>{e(h)}</th>"
|
||||
for i, h in enumerate(head)]
|
||||
t.append("</tr></thead><tbody>")
|
||||
for r in rows:
|
||||
rc = row_class(r) if row_class else ""
|
||||
t.append(f'<tr class="{rc}">' if rc else "<tr>")
|
||||
for i, x in enumerate(r):
|
||||
cls = ["num"] if i in num else []
|
||||
if cell_class and cell_class(i, x):
|
||||
cls.append(cell_class(i, x))
|
||||
c = f' class="{" ".join(cls)}"' if cls else ""
|
||||
t.append(f"<td{c}>{e(str(x))}</td>")
|
||||
t.append("</tr>")
|
||||
t.append("</tbody></table></div>")
|
||||
out.append("".join(t))
|
||||
out.append("</body></html>")
|
||||
return "\n".join(out)
|
||||
@@ -0,0 +1,243 @@
|
||||
"""The per-run report and the run-to-run comparison."""
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
|
||||
from render import Doc
|
||||
|
||||
# A change smaller than these is treated as run-to-run noise. They are a starting point, to be
|
||||
# re-calibrated from `compare --rerun-changed`, which measures the noise directly.
|
||||
NOISE = {
|
||||
"d_min": ("rel", 0.02), # 2% of the resolution limit
|
||||
"isa": ("rel", 0.05),
|
||||
"r_meas": ("rel", 0.05),
|
||||
"cc_half": ("abs", 0.005),
|
||||
"completeness": ("abs", 1.0), # percentage points
|
||||
"cell_dev_pct": ("abs", 0.2), # percentage points
|
||||
"wall_s": ("rel", 0.25), # and at least 10 s - see beyond_noise()
|
||||
}
|
||||
|
||||
|
||||
def load_run(run_dir):
|
||||
man = json.load(open(os.path.join(run_dir, "manifest.json")))
|
||||
res = json.load(open(os.path.join(run_dir, "results.json")))
|
||||
return man, res
|
||||
|
||||
|
||||
def key(r):
|
||||
return (r["arm"], r["set"])
|
||||
|
||||
|
||||
def f(x, fmt="{:.2f}", dash="-"):
|
||||
return dash if x is None else fmt.format(x)
|
||||
|
||||
|
||||
def pct(x, nd=1):
|
||||
return "-" if x is None else f"{100 * x:.{nd}f}%"
|
||||
|
||||
|
||||
def median(xs):
|
||||
xs = [x for x in xs if x is not None]
|
||||
return statistics.median(xs) if xs else None
|
||||
|
||||
|
||||
def quartiles(xs):
|
||||
xs = sorted(x for x in xs if x is not None)
|
||||
if not xs:
|
||||
return None
|
||||
if len(xs) < 4:
|
||||
return xs[0], xs[len(xs) // 2], xs[len(xs) // 2], xs[len(xs) // 2], xs[-1]
|
||||
q = statistics.quantiles(xs, n=4)
|
||||
return xs[0], q[0], q[1], q[2], xs[-1]
|
||||
|
||||
|
||||
def beyond_noise(name, a, b):
|
||||
if a is None or b is None:
|
||||
return (a is None) != (b is None)
|
||||
kind, tol = NOISE[name]
|
||||
d = abs(b - a)
|
||||
if name == "wall_s" and d < 10:
|
||||
return False
|
||||
return d > (tol * abs(a) if kind == "rel" else tol)
|
||||
|
||||
|
||||
def changes(ra, rb):
|
||||
"""What changed between two results of the same set, beyond noise."""
|
||||
out = []
|
||||
if ra["verdict"] != rb["verdict"]:
|
||||
out.append(f"verdict {ra['verdict']}->{rb['verdict']}")
|
||||
if ra.get("sgno") != rb.get("sgno"):
|
||||
out.append("space group")
|
||||
if ra.get("volume_ratio") and rb.get("volume_ratio") and \
|
||||
abs(ra["volume_ratio"] - rb["volume_ratio"]) > 0.05:
|
||||
out.append("lattice")
|
||||
for name in ("d_min", "isa", "r_meas", "cc_half", "completeness", "cell_dev_pct"):
|
||||
if beyond_noise(name, ra.get(name), rb.get(name)):
|
||||
out.append(name)
|
||||
if beyond_noise("wall_s", ra.get("wall_s"), rb.get("wall_s")):
|
||||
out.append("time")
|
||||
return out
|
||||
|
||||
|
||||
def compare_rows(res_a, res_b):
|
||||
"""[(key, ra, rb, changes)] over the sets both runs have, then those only one has."""
|
||||
a = {key(r): r for r in res_a}
|
||||
b = {key(r): r for r in res_b}
|
||||
rows = [(k, a[k], b[k], changes(a[k], b[k])) for k in sorted(a) if k in b]
|
||||
rows += [(k, a[k], None, ["only in A"]) for k in sorted(a) if k not in b]
|
||||
rows += [(k, None, b[k], ["only in B"]) for k in sorted(b) if k not in a]
|
||||
return rows
|
||||
|
||||
|
||||
def arrow(x, y, fmt=lambda v: str(v)):
|
||||
x = "-" if x is None else fmt(x)
|
||||
y = "-" if y is None else fmt(y)
|
||||
return x if x == y else f"{x} -> {y}"
|
||||
|
||||
|
||||
def compare_table(doc, rows, only_changed=True):
|
||||
head = ["set", "arm", "verdict", "space group", "d_min", "ISa", "R_meas", "CC1/2",
|
||||
"cell dev %", "time s", "beyond noise"]
|
||||
out = []
|
||||
for (arm, s), ra, rb, ch in rows:
|
||||
if only_changed and not ch:
|
||||
continue
|
||||
ra, rb = ra or {}, rb or {}
|
||||
out.append([s, arm, arrow(ra.get("verdict"), rb.get("verdict")),
|
||||
arrow(ra.get("sg"), rb.get("sg")),
|
||||
arrow(ra.get("d_min"), rb.get("d_min"), lambda v: f"{v:.2f}"),
|
||||
arrow(ra.get("isa"), rb.get("isa"), lambda v: f"{v:.1f}"),
|
||||
arrow(ra.get("r_meas"), rb.get("r_meas"), lambda v: f"{100 * v:.1f}%"),
|
||||
arrow(ra.get("cc_half"), rb.get("cc_half"), lambda v: f"{v:.3f}"),
|
||||
arrow(ra.get("cell_dev_pct"), rb.get("cell_dev_pct"), lambda v: f"{v:.2f}"),
|
||||
arrow(ra.get("wall_s"), rb.get("wall_s"), lambda v: f"{v:.0f}"),
|
||||
", ".join(ch)])
|
||||
doc.table(head, out, row_class=lambda r: "fail" if "verdict" in r[-1] else "")
|
||||
|
||||
|
||||
def crystals(rs):
|
||||
"""Rows that describe a crystal - the no-crystal controls would only pollute a distribution."""
|
||||
return [r for r in rs if "control" not in r.get("tags", [])]
|
||||
|
||||
|
||||
def scored(rs):
|
||||
return [r for r in rs if r["verdict"] in ("pass", "fail")]
|
||||
|
||||
|
||||
def rate(rs):
|
||||
s = scored(rs)
|
||||
n = sum(r["verdict"] == "pass" for r in s)
|
||||
return f"{n}/{len(s)} ({100 * n / len(s):.0f}%)" if s else "-"
|
||||
|
||||
|
||||
def build(run_dir, baseline=None):
|
||||
"""Markdown and HTML report of one run, with the delta against a baseline run if given."""
|
||||
man, res = load_run(run_dir)
|
||||
b = man["binary"]
|
||||
doc = Doc(f"Rugnux battery - {os.path.basename(os.path.normpath(run_dir))}")
|
||||
if man.get("private"):
|
||||
doc.p("PRIVATE ARM - confidential user data. This report stays in the run directory; it "
|
||||
"must not be copied into the repository, an issue or any public report.")
|
||||
doc.ul([
|
||||
f"rugnux {b.get('version', '?')}, sha256 {b['sha256'][:16]}, build flags: {b.get('flags') or 'unknown'}",
|
||||
f"runner {man['runner_git']}, host {man['host']}, {man['start']} -> {man.get('end') or 'NOT FINISHED'}",
|
||||
f"arms: {', '.join(man['arms'])}; {len(man['sets'])} sets"
|
||||
+ (f" (subset: {man['subset']})" if man.get("subset") else " (full arm)"),
|
||||
"resolution: XDS's range forced on the XDS-referenced arms" if man["options"]["xds_range"]
|
||||
else "resolution: rugnux's own choice everywhere (unforced)",
|
||||
"timing: NOT a reference - the GPU was shared" if man.get("gpu_shared")
|
||||
else "timing: GPU was not shared during the run",
|
||||
f"command options: {' '.join(man['options']['extra_args']) or '(none beyond output selection)'}",
|
||||
])
|
||||
|
||||
forced = {e["arm"] for e in man["sets"]
|
||||
if man["options"]["xds_range"] and e.get("reference") == "xds"}
|
||||
order = {arm: i for i, arm in enumerate(man["arms"])}
|
||||
|
||||
def gain(rs, arm):
|
||||
return "forced" if arm in forced else f(median(r.get("res_gain_pct") for r in rs), "{:+.1f}%")
|
||||
|
||||
doc.h(2, "Summary")
|
||||
head = ["arm", "sets", "pass", "fail", "unscored", "not run", "pass rate",
|
||||
"median res. gain", "median ISa", "median R_meas", "median time s", "total time min"]
|
||||
rows = []
|
||||
for arm in man["arms"]:
|
||||
rs = [r for r in res if r["arm"] == arm]
|
||||
cnt = {v: sum(r["verdict"] == v for r in rs) for v in ("pass", "fail", "unscored", "not_run")}
|
||||
xs = crystals(rs)
|
||||
rows.append([arm, len(rs), cnt["pass"], cnt["fail"], cnt["unscored"], cnt["not_run"], rate(rs),
|
||||
gain(xs, arm), f(median(r.get("isa") for r in xs), "{:.1f}"),
|
||||
pct(median(r.get("r_meas") for r in xs)),
|
||||
f(median(r.get("wall_s") for r in rs), "{:.0f}"),
|
||||
f(sum(r.get("wall_s") or 0 for r in rs) / 60, "{:.1f}")])
|
||||
doc.table(head, rows, num=range(1, 12))
|
||||
doc.p("Pass rate is over the scored sets (pass + fail). Resolution gain is (reference d_min - "
|
||||
"our d_min) / reference d_min: positive = finer than the deposition, or than XDS; 'forced' "
|
||||
"where XDS's resolution range was imposed, which makes it zero by construction. Medians "
|
||||
"leave out the no-crystal controls. R_meas and CC1/2 are pooled over each run's own "
|
||||
"resolution range, so they describe a run and do not rank two.")
|
||||
|
||||
doc.h(3, "By population")
|
||||
rows = []
|
||||
for arm in man["arms"]:
|
||||
rs = [r for r in res if r["arm"] == arm]
|
||||
for tag in sorted({t for r in rs for t in r.get("tags", [])}):
|
||||
sub = [r for r in rs if tag in r.get("tags", [])]
|
||||
rows.append([arm, tag, len(sub), rate(sub), gain(sub, arm)])
|
||||
doc.table(["arm", "population", "sets", "pass rate", "median res. gain"], rows, num=(2, 4))
|
||||
|
||||
doc.h(3, "Distributions (sets that merged, controls left out)")
|
||||
rows = []
|
||||
for arm in man["arms"]:
|
||||
rs = crystals(r for r in res if r["arm"] == arm)
|
||||
for name, lab, fm in (("isa", "ISa", "{:.1f}"), ("r_meas", "R_meas", None),
|
||||
("cc_half", "CC1/2", "{:.3f}"), ("res_gain_pct", "res. gain %", "{:+.1f}"),
|
||||
("wall_s", "time s", "{:.0f}")):
|
||||
if name == "res_gain_pct" and arm in forced:
|
||||
continue
|
||||
q = quartiles(r.get(name) for r in rs)
|
||||
if q:
|
||||
cells = [pct(x) if fm is None else f(x, fm) for x in q]
|
||||
rows.append([arm, lab, sum(r.get(name) is not None for r in rs)] + cells)
|
||||
doc.table(["arm", "metric", "n", "min", "q1", "median", "q3", "max"], rows, num=range(2, 8))
|
||||
|
||||
doc.h(2, "Failures")
|
||||
fails = [r for r in res if r["verdict"] == "fail"]
|
||||
doc.table(["set", "arm", "cause", "reason"],
|
||||
[[r["set"], r["arm"], r["cause"], r["reason"]] for r in fails])
|
||||
other = [r for r in res if r["verdict"] in ("unscored", "not_run")]
|
||||
if other:
|
||||
doc.p("Not scored: " + "; ".join(f"{r['set']} ({r['reason']})" for r in other))
|
||||
|
||||
if baseline:
|
||||
bman, bres = load_run(baseline)
|
||||
doc.h(2, f"Delta vs baseline {os.path.basename(os.path.normpath(baseline))}")
|
||||
doc.p(f"Baseline rugnux {bman['binary'].get('version', '?')}. Pass rates on the sets both "
|
||||
"runs have:")
|
||||
both = {key(r) for r in res} & {key(r) for r in bres}
|
||||
rows = []
|
||||
for arm in man["arms"]:
|
||||
ra = [r for r in bres if key(r) in both and r["arm"] == arm]
|
||||
rb = [r for r in res if key(r) in both and r["arm"] == arm]
|
||||
rows.append([arm, len(rb), rate(ra), rate(rb)])
|
||||
doc.table(["arm", "common sets", "baseline", "this run"], rows, num=(1,))
|
||||
doc.p("Sets whose result moved beyond noise (thresholds in report.py NOISE; confirm with "
|
||||
"`compare --rerun-changed` before believing a single-set change):")
|
||||
compare_table(doc, compare_rows(bres, res))
|
||||
|
||||
doc.h(2, "Per set")
|
||||
head = ["set", "arm", "verdict", "space group", "ref", "cell dev %", "V ratio", "d_min", "ref d_min",
|
||||
"gain", "compl %", "mult", "R_meas", "CC1/2", "ISa", "ref ISa", "idx", "time s", "note"]
|
||||
rows = []
|
||||
for r in sorted(res, key=lambda r: (order.get(r["arm"], 9), r["set"])):
|
||||
rows.append([r["set"], r["arm"], r["verdict"], r.get("sg") or "-", r.get("sg_ref") or "-",
|
||||
f(r.get("cell_dev_pct")), f(r.get("volume_ratio"), "{:.3f}"),
|
||||
f(r.get("d_min")), f(r.get("d_min_ref")), f(r.get("res_gain_pct"), "{:+.1f}%"),
|
||||
f(r.get("completeness"), "{:.1f}"), f(r.get("multiplicity"), "{:.1f}"),
|
||||
pct(r.get("r_meas")), f(r.get("cc_half"), "{:.3f}"), f(r.get("isa"), "{:.1f}"),
|
||||
f(r.get("isa_ref"), "{:.1f}"), pct(r.get("indexing_rate"), 0),
|
||||
f(r.get("wall_s"), "{:.0f}"), r.get("reason") or ""])
|
||||
doc.table(head, rows, num=range(5, 18),
|
||||
row_class=lambda row: "fail" if row[2] == "fail" else "",
|
||||
cell_class=lambda i, v: {"pass": "pass", "fail": "fail"}.get(v) if i == 2 else None)
|
||||
return doc
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Scoring one rugnux run against its reference.
|
||||
|
||||
One dataset, one verdict, one cause, decided in this order: did it run -> is the lattice right ->
|
||||
is the symmetry right -> is the merge usable. The order matters: a halved axis also makes the
|
||||
screw along it unobservable, and counting that row twice would hide the defect that cost the
|
||||
reflections.
|
||||
"""
|
||||
import itertools
|
||||
import os
|
||||
import re
|
||||
|
||||
import gemmi
|
||||
|
||||
import sgequiv
|
||||
|
||||
REPORT_LINE = re.compile(r"^([A-Z_0-9]+)=\s*(.*)$")
|
||||
|
||||
|
||||
def read_report(path):
|
||||
"""KEY= value lines of a rugnux _report.txt."""
|
||||
out = {}
|
||||
if os.path.exists(path):
|
||||
for line in open(path, errors="replace"):
|
||||
m = REPORT_LINE.match(line.rstrip("\n"))
|
||||
if m:
|
||||
out[m.group(1)] = m.group(2).strip()
|
||||
return out
|
||||
|
||||
|
||||
def fnum(s):
|
||||
try:
|
||||
return float(s.split()[0])
|
||||
except (AttributeError, ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def space_group(name, sgno):
|
||||
"""The space group in the SETTING the cell was given in: by name when there is one (I 1 2 1 and
|
||||
C 1 2 1 are both number 5, with different centring vectors), else the standard setting."""
|
||||
sg = gemmi.find_spacegroup_by_name(name) if name else None
|
||||
return sg or gemmi.find_spacegroup_by_number(int(sgno or 1))
|
||||
|
||||
|
||||
def primitive_reduced(cell, sg):
|
||||
"""Niggli-reduced PRIMITIVE cell of a cell given in space group sg (a gemmi.SpaceGroup), and
|
||||
its volume.
|
||||
|
||||
Setting-invariant: two settings of one lattice (C2 vs I2, a/c swapped, beta vs 180-beta)
|
||||
reduce to the same cell, so they compare equal."""
|
||||
gv = gemmi.GruberVector(gemmi.UnitCell(*cell), sg)
|
||||
gv.niggli_reduce()
|
||||
red = gv.cell_parameters()
|
||||
return red, gemmi.UnitCell(*red).volume
|
||||
|
||||
|
||||
def lattice_match(cell, sg, ref_cell, ref_sg):
|
||||
"""(primitive volume ratio, largest relative deviation of the reduced edges)."""
|
||||
red, vol = primitive_reduced(cell, sg)
|
||||
ref_red, ref_vol = primitive_reduced(ref_cell, ref_sg)
|
||||
edge_dev = max(abs(a - b) / b for a, b in zip(sorted(red[:3]), sorted(ref_red[:3])))
|
||||
return vol / ref_vol, edge_dev
|
||||
|
||||
|
||||
def cell_dev_pct(cell, ref):
|
||||
"""Largest relative deviation of a, b, c in percent, minimised over the six axis orders, so
|
||||
P212121 with a and c exchanged is not scored as a 6% error."""
|
||||
return min(max(abs(p[i] - ref[i]) / ref[i] * 100 for i in range(3))
|
||||
for p in itertools.permutations(cell[:3]))
|
||||
|
||||
|
||||
def point_group(sgno):
|
||||
return gemmi.find_spacegroup_by_number(int(sgno)).point_group_hm()
|
||||
|
||||
|
||||
def sg_name(sgno):
|
||||
return gemmi.find_spacegroup_by_number(int(sgno)).hm if sgno else None
|
||||
|
||||
|
||||
def judge(entry, rep, run_note):
|
||||
"""Score one set. entry is its manifest row (with 'ref'), rep its parsed report, run_note
|
||||
what the runner saw ('', 'timeout', 'exit N: message', 'no input')."""
|
||||
ref = dict(entry.get("ref") or {})
|
||||
if entry.get("ref_override"): # a reference measured by hand, replacing the file's
|
||||
ref.update(entry["ref_override"])
|
||||
r = {
|
||||
"sgno": None, "sg": None, "pg": None,
|
||||
"sgno_ref": ref.get("sgno"), "sg_ref": ref.get("sg") or sg_name(ref.get("sgno")),
|
||||
"pg_ref": point_group(ref["sgno"]) if ref.get("sgno") else None,
|
||||
"cell": None, "cell_ref": ref.get("cell"), "cell_dev_pct": None, "volume_ratio": None,
|
||||
"d_min": None, "d_min_ref": ref.get("dmin"), "res_gain_pct": None,
|
||||
"r_meas": fnum(rep.get("R_MEAS")), "cc_half": fnum(rep.get("CC_HALF")),
|
||||
"isa": fnum(rep.get("ISA")), "completeness": fnum(rep.get("COMPLETENESS")),
|
||||
"multiplicity": fnum(rep.get("MULTIPLICITY")), "i_over_sigma": fnum(rep.get("I_OVER_SIGMA")),
|
||||
"indexing_rate": fnum(rep.get("INDEXING_RATE")), "images": fnum(rep.get("IMAGES_PROCESSED")),
|
||||
"rugnux_wall_s": fnum(rep.get("WALL_TIME")), "rugnux_verdict": rep.get("VERDICT_TEXT"),
|
||||
"r_meas_ref": ref.get("r_meas"), "cc_half_ref": ref.get("cc_half"), "isa_ref": ref.get("isa"),
|
||||
"completeness_ref": ref.get("completeness"),
|
||||
}
|
||||
if rep.get("SPACE_GROUP_NUMBER"):
|
||||
r["sgno"] = int(fnum(rep["SPACE_GROUP_NUMBER"]))
|
||||
r["sg"] = rep.get("SPACE_GROUP_NAME") or sg_name(r["sgno"])
|
||||
r["pg"] = point_group(r["sgno"])
|
||||
if rep.get("UNIT_CELL_CONSTANTS"):
|
||||
r["cell"] = [float(x) for x in rep["UNIT_CELL_CONSTANTS"].split()[:6]]
|
||||
rng = (rep.get("INCLUDE_RESOLUTION_RANGE") or "").split()
|
||||
if len(rng) == 2:
|
||||
r["d_min"] = float(rng[1])
|
||||
if r["d_min"] and r["d_min_ref"]:
|
||||
r["res_gain_pct"] = round((r["d_min_ref"] - r["d_min"]) / r["d_min_ref"] * 100, 1) + 0.0 # no -0.0
|
||||
if r["cell"] and r["cell_ref"]:
|
||||
r["cell_dev_pct"] = round(cell_dev_pct(r["cell"], r["cell_ref"]), 3)
|
||||
|
||||
def verdict(v, cause, reason):
|
||||
r.update(verdict=v, cause=cause, reason=reason)
|
||||
return r
|
||||
|
||||
have_lattice = r["sgno"] is not None and r["cell"] is not None
|
||||
if entry.get("expect") == "no_lattice":
|
||||
# a no-crystal control: the right answer is to refuse
|
||||
if have_lattice:
|
||||
return verdict("fail", "false_lattice",
|
||||
f"reported a lattice ({r['sg']}) on a set with no crystal")
|
||||
return verdict("pass", None, "no lattice reported, as expected")
|
||||
if not have_lattice:
|
||||
if run_note == "no input":
|
||||
return verdict("not_run", "no_input", "input file not found")
|
||||
if run_note == "timeout":
|
||||
return verdict("fail", "timeout", "timed out")
|
||||
if re.search(r"Error reading input|Cannot open", run_note):
|
||||
return verdict("fail", "reader", run_note)
|
||||
if re.search(r"index|lattice", run_note, re.I):
|
||||
return verdict("fail", "indexing", run_note)
|
||||
return verdict("fail", "crash", run_note or "no lattice in the report")
|
||||
if not ref.get("sgno") or not ref.get("cell"):
|
||||
return verdict("unscored", "no_reference", "no reference to score against")
|
||||
|
||||
vr, edge = lattice_match(r["cell"], space_group(rep.get("SPACE_GROUP_NAME"), r["sgno"]),
|
||||
ref["cell"], space_group(ref.get("sg"), ref["sgno"]))
|
||||
r["volume_ratio"] = round(vr, 4)
|
||||
if not (0.95 < vr < 1.05 and edge < 0.02):
|
||||
if 0.45 <= vr <= 0.55:
|
||||
return verdict("fail", "lattice_halved", f"primitive volume ratio {vr:.2f}")
|
||||
if 1.9 <= vr <= 2.1:
|
||||
return verdict("fail", "lattice_doubled", f"primitive volume ratio {vr:.2f}")
|
||||
return verdict("fail", "lattice_other",
|
||||
f"primitive volume ratio {vr:.2f}, reduced edges off by {edge:.1%}")
|
||||
|
||||
# XDS never tests a screw axis, so against an XDS reference only the point group can be
|
||||
# judged - unless the group was measured by hand (sg_measured).
|
||||
by_sg = entry["arm"] == "open" or ref.get("sg_measured")
|
||||
if by_sg:
|
||||
rel = sgequiv.describe(ref["sgno"], r["sgno"])
|
||||
r["sg_relation"] = rel
|
||||
ok = rel in ("match", "hand only (needs anomalous)", "UNDECIDABLE from intensities")
|
||||
else:
|
||||
ok = r["pg"] == r["pg_ref"]
|
||||
if not ok:
|
||||
order = len(gemmi.find_spacegroup_by_number(r["sgno"]).operations().sym_ops)
|
||||
order_ref = len(gemmi.find_spacegroup_by_number(ref["sgno"]).operations().sym_ops)
|
||||
if r["pg"] == r["pg_ref"]:
|
||||
return verdict("fail", "sym_screw", f"{r['sg']} vs reference {r['sg_ref']}")
|
||||
cause = ("sym_under" if order < order_ref else
|
||||
"sym_over" if order > order_ref else "sym_other")
|
||||
return verdict("fail", cause, f"{r['sg']} vs reference {r['sg_ref']}")
|
||||
|
||||
if r["r_meas"] is not None and r["r_meas"] > 0.6:
|
||||
return verdict("fail", "merge", f"R_meas {r['r_meas']:.0%} overall")
|
||||
if r["cc_half"] is not None and r["cc_half"] < 0.5:
|
||||
return verdict("fail", "merge", f"CC1/2 {r['cc_half']:.2f} overall")
|
||||
note = ""
|
||||
if r["sgno"] != ref["sgno"]:
|
||||
note = f"{r['sg']} vs reference {r['sg_ref']} ({r.get('sg_relation') or 'screws not judged against XDS'})"
|
||||
return verdict("pass", None, note)
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Which space-group differences a diffraction experiment could actually have decided.
|
||||
|
||||
A merged dataset sees only (a) which reflections are equivalent - the rotation parts of the
|
||||
operations in their real orientation - and (b) which are systematically absent. Two space
|
||||
groups agreeing on both cannot be told apart without solving the structure. Body centring is
|
||||
the usual culprit: h+k+l=2n already forces h00: h=2n, so a 2_1 along <100> adds nothing.
|
||||
"""
|
||||
import gemmi, collections
|
||||
|
||||
_N = 7
|
||||
_HKL = [(h, k, l) for h in range(-_N, _N+1) for k in range(-_N, _N+1) for l in range(-_N, _N+1)
|
||||
if (h, k, l) != (0, 0, 0)]
|
||||
|
||||
def _signature(sg):
|
||||
ops = sg.operations()
|
||||
return (frozenset(tuple(map(tuple, op.rot)) for op in ops),
|
||||
frozenset(tuple(t) for t in ops.cen_ops),
|
||||
tuple(ops.is_systematically_absent(h) for h in _HKL))
|
||||
|
||||
_by_sig = collections.defaultdict(set)
|
||||
for _n in range(1, 231):
|
||||
_by_sig[_signature(gemmi.find_spacegroup_by_number(_n))].add(_n)
|
||||
CLASS = {n: frozenset(s) for s in _by_sig.values() for n in s if len(s) > 1}
|
||||
|
||||
def indistinguishable(a, b):
|
||||
"""True if intensities alone could never have separated these two space groups."""
|
||||
return a == b or b in CLASS.get(a, ())
|
||||
|
||||
def enantiomorphic(a, b):
|
||||
"""True if the pair differs only by hand - anomalous signal can break it, absences cannot."""
|
||||
return (indistinguishable(a, b) and a != b
|
||||
and gemmi.find_spacegroup_by_number(a).is_enantiomorphic())
|
||||
|
||||
def describe(a, b):
|
||||
if a == b: return "match"
|
||||
if enantiomorphic(a, b): return "hand only (needs anomalous)"
|
||||
if indistinguishable(a, b): return "UNDECIDABLE from intensities"
|
||||
return "REAL difference (absences differ)"
|
||||
|
||||
if __name__ == "__main__":
|
||||
seen = set()
|
||||
for n, s in sorted(CLASS.items()):
|
||||
if s in seen: continue
|
||||
seen.add(s)
|
||||
ns = sorted(s)
|
||||
print(" ", " = ".join("%d %s" % (i, gemmi.find_spacegroup_by_number(i).hm) for i in ns),
|
||||
" ->", describe(ns[0], ns[1]))
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"runs_root": "/data/battery/runs",
|
||||
"lock_file": "/data/battery/full_run.lock",
|
||||
"gpulock": "/data/tmp/gpulock",
|
||||
"arms": {
|
||||
"open": {"manifest": "open.json", "reference": "deposition", "data_root": "/home/data/nonsls/raw"},
|
||||
"inhouse": {"manifest": "inhouse.json", "reference": "xds", "data_root": "/data/rotation_test"},
|
||||
"private": {"manifest": "/data/battery/private.json", "reference": "xds", "data_root": "/data/rotation_test",
|
||||
"private": true}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user