From c9230644dfc5afb9bff5a8b36cd6dff40dea7689 Mon Sep 17 00:00:00 2001 From: Filip Leonarski Date: Fri, 25 Sep 2026 16:54:32 +0200 Subject: [PATCH] paper_plots: open arm only; --time-runs draws the time histogram from an uncontended run The time histogram now covers the open arm, like the other two figures, and can come from a run whose sets had the GPU to themselves while the quality figures use the latest run; summary.txt lists the provenance of both. The median label sits above the bars. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01D1G8gJVAy6gp1K5Dz3NE5C --- tools/battery/README.md | 13 ++++----- tools/battery/paper_plots.py | 51 ++++++++++++++++++------------------ 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/tools/battery/README.md b/tools/battery/README.md index 443e015e9..a16dfe411 100644 --- a/tools/battery/README.md +++ b/tools/battery/README.md @@ -462,12 +462,13 @@ whole range; the manifest keeps the rule used as `dmin_rule`. ## Figures for publication -`paper_plots.py RUN [RUN-REDO ...] --out DIR [--uncontended RUN]` draws three figures from a run's -`results.json`: R-free of the deposited model after the REFMAC check (`--model-check`) on the -depositor's data against Rugnux's data (same model, protocol, resolution and the depositor's free -set), the deposited high-resolution limit against Rugnux's cut (both open arm), and a histogram of -the processing time per dataset over the open and in-house arms with the median marked. Later runs -fill in sets an earlier one did not run (a `-redo`). Beside the PDF/PNG figures it writes the +`paper_plots.py RUN [RUN-REDO ...] --out DIR [--time-runs RUN ...] [--uncontended RUN]` draws three +open-arm figures from a run's `results.json`: R-free of the deposited model after the REFMAC check +(`--model-check`) on the depositor's data against Rugnux's data (same model, protocol, resolution +and the depositor's free set), the deposited high-resolution limit against Rugnux's cut, and a +histogram of the processing time per dataset with the median marked. Later runs fill in sets an +earlier one did not run (a `-redo`). `--time-runs` takes the time histogram from other runs - one +whose sets had the GPU to themselves - and `summary.txt` lists the provenance of both. Beside the PDF/PNG figures it writes the values (`rfree.dat`, `dmin.dat`, with each set's beamline; `time.dat` with set, arm, seconds, images and `gpu_others`), every excluded set with its reason (`excluded.txt`), the gnuplot script (`figures.gp`) and `summary.txt` with the runs' binary and runner provenance, the structure, diff --git a/tools/battery/paper_plots.py b/tools/battery/paper_plots.py index bd394c2c9..2ca9e5b38 100644 --- a/tools/battery/paper_plots.py +++ b/tools/battery/paper_plots.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -"""paper_plots.py -- three battery figures: R-free and high-resolution limit (deposition vs Rugnux, -open arm) and the processing time per dataset (open and in-house arms). +"""paper_plots.py -- three battery figures of the open arm: R-free and high-resolution limit +(deposition vs Rugnux) and the processing time per dataset. - tools/battery/paper_plots.py RUN [RUN ...] --out DIR [--uncontended RUN] + tools/battery/paper_plots.py RUN [RUN ...] --out DIR [--time-runs RUN [RUN ...]] [--uncontended RUN] RUN is a battery run directory (or its name under the site's runs_root); later runs fill in sets -an earlier one lacks or that failed to run there (a `-redo` of CUDA casualties). The open and -in-house arms are read; the private arm never is. Everything the figures are drawn from is written +an earlier one lacks or that failed to run there (a `-redo` of CUDA casualties). --time-runs draws +the time histogram from other runs than the two quality figures - a run whose sets had the GPU to +themselves, where the runs for the quality figures had to share it. Only the open arm is read. Everything the figures are drawn from is written beside them, so a figure can be audited and redrawn: rfree.dat, dmin.dat (value pairs with the set and its beamline), time.dat (set, arm, seconds, images, other GPU processes), excluded.txt (every set left out, with the reason), summary.txt (the runs, their binary, the counts and the timing @@ -20,8 +21,7 @@ Rugnux's data (y). Right figure: the deposited high-resolution limit (x) against (y), log axes so the low-resolution entries show. Small molecules (no deposited model) and sets whose verdict is `fail` (a wrong lattice: nothing to compare) are left out. -Third figure: a histogram of the processing time of every set of the open and in-house arms that -ran to a report, whatever its verdict, with the median marked. The time is `rugnux_wall_s`, the +Third figure: a histogram of the processing time of every open-arm set that ran to a report, whatever its verdict, with the median marked. The time is `rugnux_wall_s`, the WALL_TIME rugnux itself reports over the whole invocation (reading the images through to the written merge) - not the runner's `elapsed_s`, which also counts the wait for a GPU slot under --gpulock (`wall_s` is the same value with `elapsed_s` as the fallback). Linear 25 s bins: the @@ -32,9 +32,8 @@ page cache (a cold read of a 10 GB dataset from the workstation's HDD costs of t minute), and a run under --gpulock can share the GPU with other processes - `gpu_others` counts them as seen just before the set started. Sets that ran with a shared GPU are drawn as the lighter top segment of their bar and are never dropped; with --uncontended RUN (a run of the same sets -with the GPU to itself) summary.txt also reports how much slower they were than there. The -no-crystal controls exit without a report and so have no WALL_TIME; they and crashed sets go to -excluded.txt. +with the GPU to itself) summary.txt also reports how much slower they were than there. Sets that +crashed have no WALL_TIME and go to excluded.txt. """ import argparse import collections @@ -106,11 +105,11 @@ set xtics 100; set ytics 10 set format x "%%.0f"; set format y "%%.0f" set boxwidth %(bin)d absolute set style fill solid noborder -set label 11 "Datasets\nOpen\nIn-house" at graph 0.64, graph 0.93 left front -set label 12 "%(tn)d\n%(topen)d\n%(tinh)d" at graph 0.95, graph 0.93 right front +set label 11 "Datasets" at graph 0.64, graph 0.93 left front +set label 12 "%(tn)d" at graph 0.95, graph 0.93 right front %(tkey)s set arrow 1 from %(tmed).1f, graph 0 to %(tmed).1f, graph 1 nohead ls 1 front -set label 5 "median %(tmed).0f s" at %(tmed).1f + 0.02 * %(thi)d, graph 0.62 left front +set label 5 "median %(tmed).0f s" at %(tmed).1f + 0.02 * %(thi)d, graph 0.93 left front bin(x) = (floor(x / %(bin)d) + 0.5) * %(bin)d plot "%(out)s/time.dat" using (bin($3)):(1) smooth frequency with boxes lc rgb "#8fc0e0" title "GPU shared", \ "" using (bin($3)):($5 > 0 ? 0 : 1) smooth frequency with boxes lc rgb "#0072b2" title "GPU exclusive" @@ -118,13 +117,13 @@ plot "%(out)s/time.dat" using (bin($3)):(1) smooth frequency with boxes lc rgb " def load_rows(runs): - """Rows of the open and in-house arms, a later run's row replacing one that did not run.""" + """Rows of the open arm, a later run's row replacing one that did not run.""" rows, manifests = {}, [] for run in runs: path = run if os.path.isdir(run) else os.path.join(RUNS_ROOT, run) manifests.append((path, json.load(open(os.path.join(path, "manifest.json"))))) for r in json.load(open(os.path.join(path, "results.json"))): - if r["arm"] not in ("open", "inhouse"): + if r["arm"] != "open": continue if r["set"] not in rows or rows[r["set"]].get("exit_code"): rows[r["set"]] = r @@ -162,6 +161,8 @@ def main(): ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("runs", nargs="+", help="battery run directories or names (later ones fill in)") ap.add_argument("--out", required=True, help="output directory") + ap.add_argument("--time-runs", nargs="+", help="runs to draw the time histogram from " + "(default: the runs above)") ap.add_argument("--uncontended", help="a run of the same sets with the GPU to itself, to size the " "slowdown of the sets that shared it") a = ap.parse_args() @@ -172,14 +173,14 @@ def main(): out = os.path.abspath(a.out) rows, manifests = load_rows(a.runs) + time_rows, time_manifests = load_rows(a.time_runs) if a.time_runs else (rows, []) excluded, rfree, dmin, where, times = [], [], [], {}, [] - for s, r in sorted(rows.items()): + for s, r in sorted(time_rows.items()): if r.get("rugnux_wall_s") is None: excluded.append((s, "time", "no WALL_TIME: " + str(r.get("reason")))) else: times.append((s, r["arm"], r["rugnux_wall_s"], r.get("images"), r.get("gpu_others"))) - if r["arm"] != "open": - continue + for s, r in sorted(rows.items()): if not r.get("model"): excluded.append((s, "both", "no deposited model (small molecule or unpublished)")) continue @@ -229,8 +230,7 @@ def main(): rlo=0.05 * math.floor((min(values) - 0.01) / 0.05), rhi=0.05 * math.ceil((max(values) + 0.01) / 0.05), dlo=0.5, dhi=8.0, bin=TIME_BIN, thi=TIME_BIN * math.ceil(max(secs) / TIME_BIN), tyhi=10 * math.ceil((tallest + 1) / 10), - tmed=tmed, tn=len(times), topen=sum(arm == "open" for _, arm, _, _, _ in times), - tinh=sum(arm == "inhouse" for _, arm, _, _, _ in times)) + tmed=tmed, tn=len(times)) gp = os.path.join(out, "figures.gp") open(gp, "w").write(GNUPLOT % params) subprocess.run(["gnuplot", gp], check=True) @@ -239,7 +239,8 @@ def main(): os.path.join(out, fig)], check=True) with open(os.path.join(out, "summary.txt"), "w") as fh: - for path, m in manifests: + for path, m in manifests + [(p, dict(m, label=f"{m.get('label')} (time figure only)")) + for p, m in time_manifests]: b = m.get("binary", {}) fh.write(f"run {path}\n label {m.get('label')} binary git {b.get('source_head')} " f"dirty {b.get('source_dirty')} sha256 {b.get('sha256')}\n flags {b.get('flags')} " @@ -247,15 +248,13 @@ def main(): fh.write(f"R-free: {n} structures, {nf} facilities, {nb} beamlines; within +-{RFREE_BAND}: {within}\n" f"d_min: {dn} structures, {dnf} facilities, {dnb} beamlines; Rugnux reaches further on " f"{deeper}\n") - arm_median = lambda arm: percentile([t for _, a, t, _, _ in times if a == arm], 0.5) - queue_wait = max(rows[s]["elapsed_s"] - t for s, _, t, _, _ in times) - fh.write(f"time: {len(times)} sets (open {params['topen']}, in-house {params['tinh']}); field rugnux_wall_s " + queue_wait = max(time_rows[s]["elapsed_s"] - t for s, _, t, _, _ in times) + fh.write(f"time: {len(times)} open-arm sets; field rugnux_wall_s " f"(rugnux's WALL_TIME over the whole invocation, not the runner's elapsed_s, which adds the wait " f"for a GPU slot: here at most {queue_wait:.1f} s)\n" f" median {tmed:.1f} s, mean {sum(secs) / len(secs):.1f} s, p10 {percentile(secs, 0.1):.1f}, " f"p25 {percentile(secs, 0.25):.1f}, p75 {percentile(secs, 0.75):.1f}, p90 {percentile(secs, 0.9):.1f}, " - f"min {min(secs):.1f}, max {max(secs):.1f}; median open {arm_median('open'):.1f}, " - f"in-house {arm_median('inhouse'):.1f}\n" + f"min {min(secs):.1f}, max {max(secs):.1f}\n" f" each set ran once: a time includes reading the images from disk unless they were still in the " f"page cache\n" f" GPU shared with other processes (gpu_others > 0) on {len(shared)} sets, kept and drawn as the "