leonarski_fandClaude Opus 5
bb0648ae87
The battery is bound by reading images from disk, so instead of one battery run per setting each set now runs in all its variants in a row while its images are still in the page cache: open arm bare, then model (rugnux --model <deposited coordinates>) XDS arms bare, then xds (XDS's resolution range forced, -A where XDS was anomalous) bare (plain rugnux <input>) runs first on every arm, so its time always carries the set's disk read whichever variants are selected; each row records first_read and the report's timing table says which variant's times are cold. --variants runs a subset; --unforced is gone (--variants bare). Rows, work dirs (work/<arm>/<set>/<variant>/), compare and the report are per variant; compare pairs (arm, set, variant) on the variants both runs have on an arm, and a schema-1 run is read as one variant. The model variant takes the coordinates and published R-free from model_check's RCSB cache (site key pdb_cache) and records rugnux's R-free/R-work and the ratio; the REFMAC check is now opt-in (--model-check) and its keys moved to refmac_*. results_schema 2. model_sweep.py is retired: the model variant replaces it (its --spot/--scaling-low-resolution 50 were rugnux's defaults, so the command is the same). Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
691 lines
32 KiB
Python
691 lines
32 KiB
Python
#!/usr/bin/env python3
|
|
"""Rugnux validation battery: run, score, report, compare.
|
|
|
|
Arms and where their truth comes from:
|
|
open public PDB depositions (and a few published small-molecule sets); scored against the
|
|
deposited space group, cell and resolution. Manifest: open.json (committed).
|
|
inhouse standard test crystals measured at PSI (lysozyme, thaumatin, insulin, cytochrome C,
|
|
myoglobin) plus no-crystal controls; scored against XDS (point group only - XDS never
|
|
tests a screw axis). Manifest: inhouse.json (committed).
|
|
private user data, scored like inhouse. Its manifest lives OUTSIDE the repository (path in the
|
|
site config) and a private run gets its own run directory and report.
|
|
|
|
Usage (README.md in this directory is the full guide):
|
|
battery.py run --rugnux BIN [--label L] [--arm open,inhouse|private] [--only a,b | --tier smoke]
|
|
[--variants bare,model,xds] [--build-dir DIR] [--gpulock] [--threads N]
|
|
[--model-check] [--baseline RUN|none]
|
|
battery.py list runs under the runs root, with their state
|
|
battery.py compare RUN_A RUN_B [--all] [--rerun-changed]
|
|
battery.py report RUN --out DIR [--baseline RUN] re-render a run's report into DIR/<run>.{md,html}
|
|
battery.py abort RUN [--reason TEXT] mark an unfinished run as aborted
|
|
battery.py discover [--refresh] open arm: propose inputs for new directories
|
|
battery.py refs --arm inhouse|private [--write] re-read the XDS references from CORRECT.LP
|
|
battery.py remap --arm A MAP.json [--write] follow a rename of the data directories
|
|
|
|
The site config (data roots, runs root, lock, gpulock, private manifest, baselines) is --site, else
|
|
$JFJOCH_BATTERY_SITE, else site.json beside this script (not committed; site.example.json is).
|
|
|
|
Every set runs in each of its arm's VARIANTS (bare + model on the open arm, bare + xds on the XDS
|
|
arms), one straight after the other, so only the first reads the images from disk.
|
|
|
|
A run writes <runs_root>/<YYYYMMDD-HHMM>_<rugnux git>_<label>/ holding a copy of the binary,
|
|
manifest.json (what was run, on what, with which references), results.json (one row per set and
|
|
variant), report.md + report.html and work/<arm>/<set>/<variant>/ (log, report, cif). It is made
|
|
read-only when done.
|
|
A full-arm run (no --only / --tier) takes the site lock and refuses to start if another holds it.
|
|
A run that did not finish is never compared or reported on unless asked (--allow-incomplete).
|
|
"""
|
|
import argparse
|
|
import datetime
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import signal
|
|
import socket
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, HERE)
|
|
|
|
import inputs # noqa: E402
|
|
import report # noqa: E402
|
|
import score # noqa: E402
|
|
|
|
import model_check # noqa: E402 the deposited model (model variant); REFMAC check (--model-check)
|
|
|
|
PDB_ID = re.compile(r"^[0-9][A-Za-z0-9]{3}$")
|
|
|
|
# The variants every set of an arm runs in, by the arm's reference, in the order they run. All the
|
|
# variants of one set run back to back, so only the first reads the images from disk and the later
|
|
# ones find them in the page cache. `bare` goes first: it is in every arm, so its time is taken
|
|
# under the same condition (the set's first read) whichever variants a run selects, and it is the
|
|
# number a user sees.
|
|
VARIANTS = {"deposition": ["bare", "model"], "xds": ["bare", "xds"]}
|
|
VARIANT_DOC = {
|
|
"bare": "plain `rugnux <input>`, nothing else: what a user gets",
|
|
"model": "`rugnux --model <deposited coordinates>`: R-free of the deposited model against our merge, "
|
|
"as rugnux reports it",
|
|
"xds": "XDS's reference settings: its resolution range forced, and -A where XDS kept Friedel mates "
|
|
"apart, so both programs merge the same reflections the same way",
|
|
}
|
|
|
|
LOCAL_SITE = os.path.join(HERE, "site.json")
|
|
|
|
|
|
# ------------------------------------------------------------------ configuration and manifests
|
|
|
|
def load_site(path):
|
|
path = path or os.environ.get("JFJOCH_BATTERY_SITE") or LOCAL_SITE
|
|
if not os.path.exists(path):
|
|
sys.exit(f"no site config at {path}: copy {os.path.join(HERE, 'site.example.json')} to "
|
|
f"{LOCAL_SITE} (or anywhere, and point JFJOCH_BATTERY_SITE or --site at it) and "
|
|
"fill in this machine's paths")
|
|
site = json.load(open(path))
|
|
for arm in site["arms"].values():
|
|
arm["manifest"] = os.path.join(HERE, arm["manifest"]) # an absolute path stays as it is
|
|
return site
|
|
|
|
|
|
def load_sets(site, arm):
|
|
cfg = site["arms"][arm]
|
|
man = json.load(open(cfg["manifest"]))
|
|
for e in man["sets"]:
|
|
e["arm"] = arm
|
|
return man["sets"]
|
|
|
|
|
|
def select(site, arms, only, tier):
|
|
sets = [e for arm in arms for e in load_sets(site, arm)]
|
|
if only:
|
|
want = [s.strip() for s in only.split(",") if s.strip()]
|
|
unknown = [w for w in want if w not in {e["id"] for e in sets}]
|
|
if unknown:
|
|
sys.exit(f"not in the {'/'.join(arms)} manifest(s): {', '.join(unknown)}")
|
|
sets = [e for e in sets if e["id"] in want]
|
|
if tier:
|
|
sets = [e for e in sets if tier in e.get("tiers", {})]
|
|
return sets
|
|
|
|
|
|
# ------------------------------------------------------------------ the binary
|
|
|
|
def sha256(path):
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as fh:
|
|
for block in iter(lambda: fh.read(1 << 20), b""):
|
|
h.update(block)
|
|
return h.hexdigest()
|
|
|
|
|
|
def binary_info(path, build_dir):
|
|
"""Version and git hash as the binary itself prints them, plus the build flags from the
|
|
CMakeCache of its build tree (given, or found above the binary)."""
|
|
out = subprocess.run([path], capture_output=True, text=True, timeout=120).stdout
|
|
m = re.search(r"Version (\S+) \(git (\w+)", out)
|
|
info = {"version": m.group(1) if m else None, "git": m.group(2) if m else "unknown"}
|
|
cache = None
|
|
d = build_dir or os.path.dirname(os.path.realpath(path))
|
|
for _ in range(4):
|
|
if os.path.exists(os.path.join(d, "CMakeCache.txt")):
|
|
cache = os.path.join(d, "CMakeCache.txt")
|
|
break
|
|
d = os.path.dirname(d)
|
|
if cache:
|
|
vals = {}
|
|
for line in open(cache, errors="replace"):
|
|
m = re.match(r"^(CMAKE_BUILD_TYPE|CMAKE_CXX_FLAGS|CMAKE_HOME_DIRECTORY|JFJOCH_USE_CUDA):\w+=(.*)$",
|
|
line.strip())
|
|
if m:
|
|
vals[m.group(1)] = m.group(2)
|
|
info["cmake_cache"] = cache
|
|
info["flags"] = (f"{vals.get('CMAKE_BUILD_TYPE', '?')} CXX_FLAGS='{vals.get('CMAKE_CXX_FLAGS', '')}' "
|
|
f"CUDA={vals.get('JFJOCH_USE_CUDA', '?')}")
|
|
src = vals.get("CMAKE_HOME_DIRECTORY")
|
|
if src:
|
|
info["source"] = src
|
|
info["source_head"] = git(src, "rev-parse", "HEAD")
|
|
info["source_dirty"] = bool(git(src, "status", "--porcelain", "--untracked-files=no"))
|
|
return info
|
|
|
|
|
|
def git(d, *args):
|
|
r = subprocess.run(["git", "-C", d] + list(args), capture_output=True, text=True)
|
|
return r.stdout.strip() if r.returncode == 0 else None
|
|
|
|
|
|
def gpu_others():
|
|
"""Number of processes other than ours on the GPU right now (None if nvidia-smi is absent)."""
|
|
try:
|
|
out = subprocess.run(["nvidia-smi", "--query-compute-apps=pid", "--format=csv,noheader"],
|
|
capture_output=True, text=True, timeout=30).stdout
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
return None
|
|
return len([x for x in out.split() if x.strip().isdigit()])
|
|
|
|
|
|
# ------------------------------------------------------------------ running
|
|
|
|
def command(binary, e, variant, opts, model=None):
|
|
cmd = [binary, "-o", "p", "--no-export-unmerged"]
|
|
if opts["threads"]:
|
|
cmd += ["-N", str(opts["threads"])]
|
|
ref = e.get("ref") or {}
|
|
if variant == "xds":
|
|
# XDS's resolution range, so the two programs merge the same reflections
|
|
if ref.get("dmin"):
|
|
cmd += ["--scaling-high-resolution", f"{ref['dmin']:.3f}"]
|
|
if ref.get("dmax"):
|
|
cmd += ["--scaling-low-resolution", f"{ref['dmax']:.3f}"]
|
|
# XDS merged with FRIEDEL'S_LAW=FALSE: merge Friedel mates separately too, or the merging
|
|
# statistics are not comparable
|
|
if ref.get("anomalous"):
|
|
cmd.append("-A")
|
|
if variant == "model":
|
|
cmd += ["--model", model]
|
|
cmd += opts["extra_args"]
|
|
cmd.append(e["input_path"])
|
|
if opts["gpulock"]:
|
|
cmd = [opts["gpulock"]] + cmd
|
|
return cmd
|
|
|
|
|
|
def failure_note(log):
|
|
"""rugnux's own last error message, without the spdlog timestamp and severity prefix."""
|
|
try:
|
|
lines = [l.strip() for l in open(log, errors="replace") if l.strip()]
|
|
except OSError:
|
|
return ""
|
|
for l in reversed(lines):
|
|
if "[error]" in l or "failed" in l.lower():
|
|
return l.split("] ")[-1][:300]
|
|
return lines[-1].split("] ")[-1][:300] if lines else ""
|
|
|
|
|
|
def deposited_model(e, opts):
|
|
"""(deposition record from model_check's RCSB cache, None) for the entry a set belongs to, or
|
|
(None, why not). A set id may carry a suffix (6h2p_native); the entry is the part before it."""
|
|
pdb = e["id"].split("_")[0].lower()
|
|
if not PDB_ID.match(pdb):
|
|
return None, "no model: not a PDB entry (small molecule or unpublished set)"
|
|
meta = model_check.deposition(pdb, opts["pdb_cache"])
|
|
if meta is None:
|
|
return None, f"no model: coordinates of {pdb} could not be downloaded"
|
|
return meta, None
|
|
|
|
|
|
def run_one(run_dir, binary, e, variant, first_read, opts):
|
|
wd = os.path.join(run_dir, "work", e["arm"], e["id"], variant)
|
|
os.makedirs(wd)
|
|
row = {"set": e["id"], "arm": e["arm"], "variant": variant, "first_read": first_read,
|
|
"tags": e.get("tags", []), "input": e["input_path"], "gpu_others": gpu_others(),
|
|
"rfree_deposited": None, "rfree_ratio": None, "cmd": None, "exit_code": None,
|
|
"elapsed_s": None, "wall_s": None}
|
|
row.update({k: None for k in REFMAC_KEYS})
|
|
if not os.path.exists(e["input_path"]):
|
|
row.update(score.judge(e, {}, "no input"))
|
|
return row
|
|
meta = None
|
|
if variant == "model":
|
|
meta, why = deposited_model(e, opts)
|
|
if meta is None:
|
|
row.update(score.judge(e, {}, why))
|
|
return row
|
|
row["rfree_deposited"] = meta["rfree"]
|
|
cmd = command(binary, e, variant, opts, meta and meta["xyz"])
|
|
row["cmd"] = " ".join(cmd)
|
|
t0 = time.time()
|
|
note = ""
|
|
with open(os.path.join(wd, "run.log"), "w") as log:
|
|
try:
|
|
rc = subprocess.run(cmd, cwd=wd, stdout=log, stderr=subprocess.STDOUT,
|
|
timeout=opts["timeout"]).returncode
|
|
except subprocess.TimeoutExpired:
|
|
rc, note = None, "timeout"
|
|
row["elapsed_s"] = round(time.time() - t0, 1)
|
|
row["exit_code"] = rc
|
|
if rc:
|
|
note = f"exit {rc}: {failure_note(os.path.join(wd, 'run.log'))}"
|
|
row.update(score.judge(e, score.read_report(os.path.join(wd, "p_report.txt")), note))
|
|
# rugnux's own WALL_TIME where it wrote one: the runner's clock also counts the wait for a
|
|
# GPU slot under --gpulock
|
|
row["wall_s"] = row["rugnux_wall_s"] if row["rugnux_wall_s"] is not None else row["elapsed_s"]
|
|
if row["rfree"] and row["rfree_deposited"]:
|
|
row["rfree_ratio"] = round(row["rfree"] / row["rfree_deposited"], 4)
|
|
if opts["model_check"] and e["arm"] == "open" and variant == "bare":
|
|
row.update(check_model(e, wd))
|
|
return row
|
|
|
|
|
|
# what the REFMAC model check (model_check.py, --model-check) adds to an open-arm bare row
|
|
REFMAC_KEYS = ("refmac_rfree", "refmac_rwork", "refmac_rfree_depflags", "refmac_rfree_depdata",
|
|
"refmac_rfree_ratio", "refmac_status", "refmac_reason")
|
|
|
|
|
|
def check_model(e, wd):
|
|
"""R-free of the deposited model against this run's merge, by REFMAC (model_check.py)."""
|
|
out = {k: None for k in REFMAC_KEYS}
|
|
pdb = e["id"].split("_")[0]
|
|
mtz = os.path.join(wd, "p.mtz")
|
|
if not PDB_ID.match(pdb):
|
|
return dict(out, refmac_status="skipped", refmac_reason="not a PDB entry")
|
|
if not os.path.exists(mtz):
|
|
return dict(out, refmac_status="skipped", refmac_reason="no merged MTZ")
|
|
try:
|
|
res = model_check.check(mtz, pdb, os.path.join(wd, "model_check"))
|
|
except Exception as ex: # a failing check must not lose the set's processing result
|
|
return dict(out, refmac_status="error", refmac_reason=f"{type(ex).__name__}: {ex}"[:300])
|
|
out.update({"refmac_" + k: res.get(k) for k in ("rfree", "rwork", "rfree_depflags", "rfree_depdata")},
|
|
refmac_status=res.get("status"), refmac_reason=res.get("reason"))
|
|
# The fair ratio: our merge and the depositor's structure factors, both scored on the
|
|
# depositor's free set by the same protocol. Our own free set was mostly work reflections
|
|
# for the depositor, so R_free on it reads low.
|
|
if out["refmac_rfree_depflags"] and out["refmac_rfree_depdata"]:
|
|
out["refmac_rfree_ratio"] = round(out["refmac_rfree_depflags"] / out["refmac_rfree_depdata"], 4)
|
|
return out
|
|
|
|
|
|
def make_read_only(path):
|
|
for dp, dirs, files in os.walk(path):
|
|
for fn in files:
|
|
p = os.path.join(dp, fn)
|
|
if not os.path.islink(p):
|
|
os.chmod(p, os.stat(p).st_mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
|
|
for dp, dirs, files in os.walk(path, topdown=False):
|
|
os.chmod(dp, os.stat(dp).st_mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
|
|
|
|
|
|
def write_json(path, obj):
|
|
with open(path + ".tmp", "w") as fh:
|
|
json.dump(obj, fh, indent=1)
|
|
os.replace(path + ".tmp", path)
|
|
|
|
|
|
def write_manifest(path, man):
|
|
"""A manifest with one set per line, so a diff of it reads set by set."""
|
|
lines = [json.dumps(e) for e in man["sets"]]
|
|
head = json.dumps({k: v for k, v in man.items() if k != "sets"})[:-1]
|
|
with open(path, "w") as fh:
|
|
fh.write(head + ', "sets": [\n ' + ",\n ".join(lines) + "\n]}\n")
|
|
|
|
|
|
def write_report(run_dir, baseline=None, out_dir=None, allow_incomplete=False):
|
|
"""report.md and report.html (report_PRIVATE.* for a private run) in the run directory, or
|
|
<run name>.md/.html (with _PRIVATE) in out_dir."""
|
|
man = json.load(open(os.path.join(run_dir, "manifest.json")))
|
|
name = os.path.basename(os.path.normpath(run_dir)) if out_dir else "report"
|
|
base = os.path.join(out_dir or run_dir, name + ("_PRIVATE" if man.get("private") else ""))
|
|
doc = report.build(run_dir, baseline, allow_incomplete=allow_incomplete or not out_dir)
|
|
open(base + ".md", "w").write(doc.markdown())
|
|
open(base + ".html", "w").write(doc.html())
|
|
return base + ".html"
|
|
|
|
|
|
def run_sets(site, sets, arms, binary_src, build_dir, label, opts, subset, baseline=None, lock=None):
|
|
"""Run the given manifest rows into a new run directory. Returns the directory."""
|
|
private = any(site["arms"][a].get("private") for a in arms)
|
|
if private and len(arms) > 1:
|
|
sys.exit("the private arm runs on its own: its results must not share a run or a report")
|
|
if baseline:
|
|
report.check_baseline(baseline, private)
|
|
lock_fh = None
|
|
if lock:
|
|
os.makedirs(os.path.dirname(lock), exist_ok=True)
|
|
lock_fh = open(lock, "w")
|
|
try:
|
|
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except BlockingIOError:
|
|
sys.exit(f"another full battery run holds {lock}; refusing to start a second one")
|
|
|
|
info = binary_info(binary_src, build_dir)
|
|
label = label or info.get("version") or "run"
|
|
stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M")
|
|
run_dir = os.path.join(opts["runs_root"],
|
|
f"{stamp}_{info['git']}_{label}" + ("_private" if private else ""))
|
|
os.makedirs(run_dir) # never reuse or overwrite a run directory
|
|
binary = os.path.join(run_dir, "bin", "rugnux")
|
|
os.makedirs(os.path.dirname(binary))
|
|
shutil.copy2(binary_src, binary)
|
|
info.update(copied_from=os.path.realpath(binary_src), sha256=sha256(binary))
|
|
|
|
for e in sets:
|
|
e["reference"] = site["arms"][e["arm"]]["reference"]
|
|
e.setdefault("input_path", os.path.join(site["arms"][e["arm"]]["data_root"], e["input"]))
|
|
variants = {arm: [v for v in VARIANTS[site["arms"][arm]["reference"]] if v in opts["variants"]]
|
|
for arm in arms}
|
|
man = {"label": label, "arms": arms, "private": private, "subset": subset, "baseline": baseline,
|
|
"variants": variants, "variant_doc": {v: VARIANT_DOC[v] for v in opts["variants"]},
|
|
"binary": info,
|
|
"runner_git": git(HERE, "rev-parse", "--short", "HEAD") or "unknown",
|
|
"runner_dirty": bool(git(HERE, "status", "--porcelain", "--", ".")),
|
|
"host": socket.gethostname(), "pid": os.getpid(), "start": datetime.datetime.now().isoformat(timespec="seconds"),
|
|
"end": None, "complete": False, "results_schema": report.RESULTS_SCHEMA, "options": {k: v for k, v in opts.items() if k != "runs_root"},
|
|
"gpu_shared": bool(opts["gpulock"]), "sets": sets}
|
|
write_json(os.path.join(run_dir, "manifest.json"), man)
|
|
print(f"run directory {run_dir}", flush=True)
|
|
|
|
results = []
|
|
todo = sum(len(variants[e["arm"]]) for e in sets)
|
|
# a kill by SIGTERM is recorded like Ctrl-C: the run directory then says it was aborted
|
|
signal.signal(signal.SIGTERM, interrupted)
|
|
try:
|
|
for e in sets:
|
|
# the variants of one set back to back: only the first rugnux reads the images from disk
|
|
first_read = True
|
|
for v in variants[e["arm"]]:
|
|
row = run_one(run_dir, binary, e, v, first_read, opts)
|
|
first_read = first_read and row["cmd"] is None
|
|
results.append(row)
|
|
if row.get("gpu_others"):
|
|
man["gpu_shared"] = True
|
|
write_json(os.path.join(run_dir, "results.json"), results)
|
|
print(f"[{len(results)}/{todo}] {e['arm']:8s} {e['id']:24s} {v:6s} {row['verdict']:8s} "
|
|
f"{row.get('sg') or '-':12s} d={report.f(row.get('d_min'))} "
|
|
f"t={report.f(row.get('wall_s'), '{:.0f}')}s {row.get('reason') or ''}", flush=True)
|
|
except KeyboardInterrupt:
|
|
mark_aborted(run_dir, f"interrupted after {len(results)} of {todo} rows")
|
|
sys.exit(f"aborted; {run_dir} is marked as such")
|
|
|
|
man.update(end=datetime.datetime.now().isoformat(timespec="seconds"), complete=True)
|
|
write_json(os.path.join(run_dir, "manifest.json"), man)
|
|
print(f"report {write_report(run_dir, baseline)}", flush=True)
|
|
make_read_only(run_dir)
|
|
if lock_fh:
|
|
lock_fh.close()
|
|
return run_dir
|
|
|
|
|
|
def interrupted(*_):
|
|
raise KeyboardInterrupt
|
|
|
|
|
|
def mark_aborted(run_dir, reason):
|
|
man = json.load(open(os.path.join(run_dir, "manifest.json")))
|
|
man["aborted"] = {"at": datetime.datetime.now().isoformat(timespec="seconds"), "reason": reason}
|
|
write_json(os.path.join(run_dir, "manifest.json"), man)
|
|
make_read_only(run_dir)
|
|
|
|
|
|
def default_baseline(site, arms, given):
|
|
"""--baseline, else the site's persisted baseline for this kind of run ('none' = no delta)."""
|
|
if given:
|
|
return None if given == "none" else given
|
|
private = any(site["arms"][a].get("private") for a in arms)
|
|
return site.get("baseline_private" if private else "baseline") or None
|
|
|
|
|
|
# ------------------------------------------------------------------ subcommands
|
|
|
|
def cmd_run(a, site):
|
|
arms = [x.strip() for x in a.arm.split(",")]
|
|
for arm in arms:
|
|
if arm not in site["arms"]:
|
|
sys.exit(f"unknown arm {arm}; the site config has {', '.join(site['arms'])}")
|
|
sets = select(site, arms, a.only, a.tier)
|
|
if not sets:
|
|
sys.exit("nothing selected")
|
|
subset = a.only or (f"tier {a.tier}" if a.tier else None)
|
|
known = [v for vs in VARIANTS.values() for v in vs]
|
|
want = [v.strip() for v in a.variants.split(",")] if a.variants else known
|
|
if set(want) - set(known):
|
|
sys.exit(f"unknown variant(s) {', '.join(sorted(set(want) - set(known)))}; "
|
|
f"there are {', '.join(dict.fromkeys(known))}")
|
|
if not any(v in want for arm in arms for v in VARIANTS[site["arms"][arm]["reference"]]):
|
|
sys.exit(f"none of the variants {a.variants} runs on the {'/'.join(arms)} arm(s)")
|
|
opts = {"threads": a.threads, "variants": [v for v in dict.fromkeys(known) if v in want],
|
|
"timeout": a.timeout, "model_check": a.model_check and "open" in arms,
|
|
"pdb_cache": site.get("pdb_cache") or model_check.DEFAULT_CACHE,
|
|
"gpulock": site["gpulock"] if a.gpulock else None, "extra_args": a.extra.split(),
|
|
"runs_root": a.runs_root or site["runs_root"]}
|
|
if a.gpulock and not site.get("gpulock"):
|
|
sys.exit("--gpulock: the site config names no gpulock")
|
|
run_sets(site, sets, arms, a.rugnux, a.build_dir, a.label, opts, subset,
|
|
default_baseline(site, arms, a.baseline), lock=None if subset else site["lock_file"])
|
|
|
|
|
|
def cmd_list(a, site):
|
|
root = a.runs_root or site["runs_root"]
|
|
for d in sorted(os.listdir(root)):
|
|
run = os.path.join(root, d)
|
|
if not os.path.exists(os.path.join(run, "manifest.json")):
|
|
continue
|
|
man = json.load(open(os.path.join(run, "manifest.json")))
|
|
state = report.run_state(man)
|
|
n = len(json.load(open(os.path.join(run, "results.json")))) \
|
|
if os.path.exists(os.path.join(run, "results.json")) else 0
|
|
mark = " <- baseline" if run in (site.get("baseline"), site.get("baseline_private")) else ""
|
|
print(f"{d:48s} {state:10s} {n:3d}/{report.expected_rows(man):3d} rows "
|
|
f"{man.get('subset') or 'full arm':20.20s} {','.join(man['arms'])}{mark}")
|
|
|
|
|
|
def cmd_abort(a, site):
|
|
man = json.load(open(os.path.join(a.run, "manifest.json")))
|
|
state = report.run_state(man)
|
|
if state not in ("unfinished", "running") or (state == "running" and not a.force):
|
|
sys.exit(f"{a.run} is {state}; only an unfinished run can be marked aborted "
|
|
"(a live one needs --force)")
|
|
mark_aborted(a.run, a.reason)
|
|
print(f"{a.run}: marked aborted ({a.reason})")
|
|
|
|
|
|
def cmd_compare(a, site):
|
|
man_a, res_a = report.load_run(a.run_a, a.allow_incomplete)
|
|
man_b, res_b = report.load_run(a.run_b, a.allow_incomplete)
|
|
rows = report.compare_rows(res_a, res_b)
|
|
doc = report.Doc(f"{os.path.basename(os.path.normpath(a.run_a))} vs "
|
|
f"{os.path.basename(os.path.normpath(a.run_b))}")
|
|
doc.p(f"A: rugnux {man_a['binary'].get('version')} ({man_a['binary']['sha256'][:12]}); "
|
|
f"B: rugnux {man_b['binary'].get('version')} ({man_b['binary']['sha256'][:12]}). "
|
|
f"{len(rows)} set/variant rows, {sum(1 for r in rows if r[3])} changed beyond noise. "
|
|
f"Variants compared: {report.pairs_text(report.common_variants(res_a, res_b))} "
|
|
"(the ones both runs have).")
|
|
report.compare_table(doc, rows, only_changed=not a.all)
|
|
print(doc.markdown())
|
|
if not a.rerun_changed:
|
|
return
|
|
# Rerun the changed sets with A's own binary and options: a set that moves again under the
|
|
# same binary was noise; one that reproduces A was changed by B for real.
|
|
changed = {k for k, ra, rb, ch in rows if ra and rb and [c for c in ch if c != "time"]}
|
|
if not changed:
|
|
print("nothing changed beyond noise; no rerun")
|
|
return
|
|
sets = [e for e in man_a["sets"] if (e["arm"], e["id"]) in {k[:2] for k in changed}]
|
|
opts = dict(man_a["options"], runs_root=a.runs_root or site["runs_root"],
|
|
variants=[v for v in VARIANT_DOC if v in {k[2] for k in changed}])
|
|
opts["gpulock"] = opts["gpulock"] and site.get("gpulock")
|
|
opts.setdefault("pdb_cache", site.get("pdb_cache") or model_check.DEFAULT_CACHE)
|
|
label = "rerunA-" + man_a["label"]
|
|
rerun = run_sets(site, sets, man_a["arms"], os.path.join(a.run_a, "bin", "rugnux"), None, label,
|
|
opts, ",".join(e["id"] for e in sets))
|
|
_, res_a2 = report.load_run(rerun)
|
|
again = {k: ch for k, ra, ra2, ch in report.compare_rows(res_a, res_a2) if ra and ra2}
|
|
doc = report.Doc("Changed sets, rerun with A's binary")
|
|
rows_out = []
|
|
for k, ra, rb, ch in rows:
|
|
if k not in changed:
|
|
continue
|
|
noise = [c for c in again.get(k, []) if c != "time"]
|
|
rows_out.append([k[1], k[0], k[2], ", ".join(c for c in ch if c != "time"),
|
|
", ".join(noise) or "-",
|
|
"NOISE - A itself moved" if noise else "REAL - A reproduces"])
|
|
doc.table(["set", "arm", "variant", "A -> B", "A -> A again", "reading"], rows_out)
|
|
doc.p(f"rerun directory: {rerun}")
|
|
print(doc.markdown())
|
|
|
|
|
|
def cmd_report(a, site):
|
|
# a finished run directory is read-only, so a re-render goes elsewhere, named after the run
|
|
man, _ = report.load_run(a.run, a.allow_incomplete)
|
|
repo = git(HERE, "rev-parse", "--show-toplevel")
|
|
out = os.path.realpath(a.out)
|
|
if man.get("private") and repo and (out + os.sep).startswith(os.path.realpath(repo) + os.sep):
|
|
sys.exit("a private run's report must not be written into the repository")
|
|
os.makedirs(a.out, exist_ok=True)
|
|
print(write_report(a.run, a.baseline, a.out, a.allow_incomplete))
|
|
|
|
|
|
def cmd_discover(a, site):
|
|
"""Propose an input for every open-arm directory the manifest does not have yet; with
|
|
--refresh, also show where today's discovery disagrees with the manifest's (unpinned) input."""
|
|
cfg = site["arms"]["open"]
|
|
root = cfg["data_root"]
|
|
by_dir = {} # a directory can hold several sets (6h2p_native, 6h2p_1p89A)
|
|
for e in load_sets(site, "open"):
|
|
by_dir.setdefault(e["input"].split("/")[0], []).append(e)
|
|
for d in sorted(os.listdir(root)):
|
|
if d.startswith(".") or not os.path.isdir(os.path.join(root, d)):
|
|
continue
|
|
sets = by_dir.get(d, [])
|
|
if (sets and not a.refresh) or len(sets) > 1 or (sets and sets[0].get("pinned")):
|
|
continue
|
|
path, fmt, note = inputs.find_input(os.path.join(root, d))
|
|
rel = os.path.relpath(path, root) if path else None
|
|
if not sets:
|
|
print(f"NEW {d:10s} {rel} [{note}]")
|
|
elif rel != sets[0]["input"]:
|
|
print(f"DIFF {d:10s} manifest {sets[0]['input']}\n {'':10s} discover {rel} [{note}]")
|
|
for d in sorted(set(by_dir) - set(os.listdir(root))):
|
|
print(f"GONE {d:10s} in the manifest, not on disk")
|
|
|
|
|
|
def cmd_refs(a, site):
|
|
"""Re-read every set's XDS reference from the CORRECT.LP beside its input."""
|
|
cfg = site["arms"][a.arm]
|
|
if cfg["reference"] != "xds":
|
|
sys.exit(f"the {a.arm} arm is not referenced to XDS")
|
|
man = json.load(open(cfg["manifest"]))
|
|
for e in man["sets"]:
|
|
lp = os.path.join(cfg["data_root"], os.path.dirname(e["input"]), "CORRECT.LP")
|
|
if not os.path.exists(lp):
|
|
print(f"{e['id']:24s} no CORRECT.LP")
|
|
continue
|
|
new = inputs.parse_correct_lp(lp)
|
|
old = e.get("ref") or {}
|
|
diff = [f"{k} {old.get(k)} -> {new.get(k)}" for k in sorted(set(old) | set(new))
|
|
if old.get(k) != new.get(k)]
|
|
if diff:
|
|
print(f"{e['id']:24s} " + "; ".join(diff))
|
|
e["ref"] = new
|
|
if a.write:
|
|
write_manifest(cfg["manifest"], man)
|
|
|
|
|
|
def cmd_remap(a, site):
|
|
"""Follow a rename of data directories. MAP is a JSON object {old: new}: a string renames the
|
|
set of that id and, wherever it is the first directory of an input, that directory; an object
|
|
{"id": ..., "input": ...} gives one set's new id and input outright. A renamed set's old id is
|
|
kept under "aliases", so runs made before the rename still compare set by set."""
|
|
cfg = site["arms"][a.arm]
|
|
man = json.load(open(cfg["manifest"]))
|
|
mp = json.load(open(a.map))
|
|
aliases = man.setdefault("aliases", {})
|
|
for e in man["sets"]:
|
|
new = mp.get(e["id"])
|
|
head, _, rest = e["input"].partition("/")
|
|
if isinstance(new, dict):
|
|
new_id, new_input = new.get("id", e["id"]), new.get("input", e["input"])
|
|
else:
|
|
new_id = new or e["id"]
|
|
new_input = (mp[head] if isinstance(mp.get(head), str) else head) + "/" + rest
|
|
if (new_id, new_input) == (e["id"], e["input"]):
|
|
continue
|
|
there = os.path.exists(os.path.join(cfg["data_root"], new_input))
|
|
print(f"{e['id']:24s} -> {new_id:24s} {new_input}" + ("" if there else " NOT FOUND"))
|
|
if new_id != e["id"]:
|
|
for old in [k for k, v in aliases.items() if v == e["id"]]:
|
|
aliases[old] = new_id
|
|
aliases[e["id"]] = new_id
|
|
e["id"], e["input"] = new_id, new_input
|
|
if a.write:
|
|
write_manifest(cfg["manifest"], man)
|
|
|
|
|
|
def load_aliases(site):
|
|
"""{arm: {old set id: current id}} from the manifests, applied when runs are compared."""
|
|
out = {}
|
|
for arm, cfg in site["arms"].items():
|
|
if os.path.exists(cfg["manifest"]):
|
|
out[arm] = json.load(open(cfg["manifest"])).get("aliases", {})
|
|
return out
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
|
|
ap.add_argument("--site", help="site config (default: $JFJOCH_BATTERY_SITE, else site.json "
|
|
"beside this script)")
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
|
|
r = sub.add_parser("run")
|
|
r.add_argument("--rugnux", required=True)
|
|
r.add_argument("--label", help="short name, part of the run directory's name "
|
|
"(default: the rugnux version)")
|
|
r.add_argument("--arm", default="open,inhouse", help="comma-separated (default: open,inhouse)")
|
|
r.add_argument("--only", help="comma-separated set ids")
|
|
r.add_argument("--tier", help="named subset from the manifests, e.g. smoke")
|
|
r.add_argument("--build-dir", help="build tree whose CMakeCache.txt records the flags")
|
|
r.add_argument("--gpulock", action="store_true",
|
|
help="queue each rugnux behind the site's GPU lock (timing is then not a reference)")
|
|
r.add_argument("--threads", type=int, default=0, help="-N for rugnux (default: its own choice)")
|
|
r.add_argument("--variants", help="comma-separated subset of bare,model,xds (default: all of "
|
|
"each arm's variants; e.g. 'bare' for what a user gets)")
|
|
r.add_argument("--timeout", type=int, default=10800, help="per rugnux run, in s")
|
|
r.add_argument("--model-check", action="store_true",
|
|
help="open arm: also score each bare merge against the deposited model with "
|
|
"REFMAC (model_check.py; needs CCP4)")
|
|
r.add_argument("--extra", default="", help="extra rugnux arguments, applied to every set")
|
|
r.add_argument("--baseline", help="earlier run directory to report the delta against (default: "
|
|
"the site's persisted baseline; 'none' for no delta)")
|
|
r.add_argument("--runs-root")
|
|
|
|
c = sub.add_parser("compare")
|
|
c.add_argument("run_a")
|
|
c.add_argument("run_b")
|
|
c.add_argument("--all", action="store_true", help="list unchanged sets too")
|
|
c.add_argument("--rerun-changed", action="store_true",
|
|
help="rerun the changed sets with A's saved binary to separate real change from noise")
|
|
c.add_argument("--runs-root")
|
|
c.add_argument("--allow-incomplete", action="store_true",
|
|
help="compare a run that did not finish (aborted, killed or still running)")
|
|
|
|
p = sub.add_parser("report")
|
|
p.add_argument("run")
|
|
p.add_argument("--baseline")
|
|
p.add_argument("--out", required=True, help="directory for the re-rendered report")
|
|
p.add_argument("--allow-incomplete", action="store_true",
|
|
help="render a run that did not finish; the report says so")
|
|
|
|
ls = sub.add_parser("list")
|
|
ls.add_argument("--runs-root")
|
|
|
|
ab = sub.add_parser("abort")
|
|
ab.add_argument("run")
|
|
ab.add_argument("--reason", default="aborted by hand")
|
|
ab.add_argument("--force", action="store_true", help="even if its runner still seems alive")
|
|
|
|
d = sub.add_parser("discover")
|
|
d.add_argument("--refresh", action="store_true")
|
|
|
|
m = sub.add_parser("remap")
|
|
m.add_argument("--arm", required=True)
|
|
m.add_argument("map", help="JSON file {old: new}")
|
|
m.add_argument("--write", action="store_true")
|
|
|
|
f = sub.add_parser("refs")
|
|
f.add_argument("--arm", required=True)
|
|
f.add_argument("--write", action="store_true")
|
|
|
|
a = ap.parse_args()
|
|
site = load_site(a.site)
|
|
report.ALIASES = load_aliases(site)
|
|
{"run": cmd_run, "list": cmd_list, "compare": cmd_compare, "report": cmd_report,
|
|
"abort": cmd_abort, "discover": cmd_discover, "refs": cmd_refs, "remap": cmd_remap}[a.cmd](a, site)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|