Battery: the time figures can count more arms, and a second one zooms on the first 60 s
paper_plots.py --time-arms (default open, so the paper figures do not change) widens the time histogram to e.g. the open and in-house arms; time_zoom is the same histogram over 0-60 s in 5 s bins, with the sets beyond it counted in the label. Also brings 8xtg's accepted-alternative note in open.json up to the zone read at the other twin laws' fraction. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01K5K8jvPPbmCrbqnWkddTuB
This commit is contained in:
Claude Opus 5.5
@@ -108,7 +108,7 @@
|
||||
{"id": "8xbp", "input": "8xbp/8xbp/data/puck2872_3_1_master.h5", "wavelength": 0.97856, "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [148.29, 50.78, 60.21, 90.0, 92.33, 90.0], "dmin": 1.99}, "ref_override": {"cell": [148.736, 51.777, 60.181, 90.0, 92.36, 90.0]}, "ref_override_why": "the uploaded sweep is a different crystal from the one the deposited cell describes: master data_collection_date 2023-06-21, deposited _diffrn_detector.pdbx_collection_date 2023-06-23, and the deposited b = 50.78 is 2.0% from the b these images give. The override is the cell DIALS 3.29 indexes de novo on this master, b = 51.777(11); the deposited space group and d_min are kept", "tags": ["h5", "monoclinic"]},
|
||||
{"id": "8xte", "input": "8xte/1101/G2_1_00001.cbf", "wavelength": 0.97853, "ref": {"sg": "P 32", "sgno": 145, "cell": [208.77, 208.77, 67.22, 90.0, 90.0, 120.0], "dmin": 1.99}, "ref_alternatives": [{"sg": "P 31 2 1", "sgno": 152, "why": "The evidence favours the higher group here. The twin-immune centric zone of the added two-folds - reflections the higher group makes centric, which are their own twin mates and so cannot be made to read centric by a merohedral twin law - gives <|E^2-1|> = 0.946 +/- 0.012 against a centric expectation of 0.968 and an acentric 0.736, at +707 nats. Re-refinement on a shared free set with the twin law removed from both sides favours P 3_2 2 1 (0.2177/0.2322) over P 3_2 (0.2460/0.2612), and the deposited entry's published R values reproduce only with an undeclared twin law h,-h-k,-l at alpha = 0.50, which is itself a 321-symmetric target. No refinement R can close the question in principle, because a merohedral twin at exactly alpha = 0.5 and true 321 predict identical intensities; the case rests on the centric zone. Both answers are accepted, ours being the better supported"}], "tags": ["cbf", "trigonal"]},
|
||||
{"id": "8xtf", "input": "8xtf/1102/puck03_12_1_master.h5", "wavelength": 0.97918, "ref": {"sg": "H 3 2", "sgno": 155, "cell": [211.75, 211.75, 67.42, 90.0, 90.0, 120.0], "dmin": 2.13}, "tags": ["h5", "trigonal"]},
|
||||
{"id": "8xtg", "input": "8xtg/1100/mpag15_1_00001.cbf", "wavelength": 0.97853, "ref": {"sg": "P 32", "sgno": 145, "cell": [199.54, 199.54, 67.15, 90.0, 90.0, 120.0], "dmin": 2.0}, "ref_alternatives": [{"sg": "P 31 2 1", "sgno": 152, "why": "The evidence favours the DEPOSITION here, and this row must not be read as the 8xte one. Every correlation-based instrument we have - our own operator correlations, POINTLESS (0.85 on our P1 merge) - reads point group 321, but our own twin-immune centric-zone test, the only one that separates real symmetry from pseudo-symmetry, reads <|E^2-1|> = 0.869 at -44.9 nats, between the two expectations and on the wrong side, i.e. AGAINST the promotion, with an L-test twin fraction of 0.20-0.26. Whether this crystal is partially twinned or purely pseudo-symmetric is not established. Both answers are accepted, the deposition being the better supported"}], "tags": ["cbf", "trigonal"]},
|
||||
{"id": "8xtg", "input": "8xtg/1100/mpag15_1_00001.cbf", "wavelength": 0.97853, "ref": {"sg": "P 32", "sgno": 145, "cell": [199.54, 199.54, 67.15, 90.0, 90.0, 120.0], "dmin": 2.0}, "ref_alternatives": [{"sg": "P 31 2 1", "sgno": 152, "why": "The evidence favours the DEPOSITION here, and this row must not be read as the 8xte one. Every correlation-based instrument we have - our own operator correlations, POINTLESS (0.85 on our P1 merge) - reads point group 321, but our own twin-immune centric-zone test, the only one that separates real symmetry from pseudo-symmetry, read <|E^2-1|> = 0.869 at -44.9 nats against the promotion when it assumed an untwinned zone; read at the fraction of the crystal's other twin laws (rc174) it now favours the promotion, with an L-test twin fraction of 0.20-0.26. Whether this crystal is partially twinned or purely pseudo-symmetric is not established. Both answers are accepted, the deposition being the better supported"}], "tags": ["cbf", "trigonal"]},
|
||||
{"id": "8y74", "input": "8y74/1_1/1_1_master.h5", "wavelength": 0.97918, "ref": {"sg": "C 1 2 1", "sgno": 5, "cell": [125.782, 76.553, 87.143, 90.0, 92.449, 90.0], "dmin": 1.9}, "tags": ["h5", "monoclinic"]},
|
||||
{"id": "8ys9", "input": "8ys9/8YS9_xray-data_8ys9/data/SDS_09_18112_master.h5", "wavelength": 0.97957, "ref": {"sg": "P 21 21 21", "sgno": 19, "cell": [71.04, 77.68, 83.16, 90.0, 90.0, 90.0], "dmin": 1.46}, "tags": ["h5", "orthorhombic"]},
|
||||
{"id": "9b22", "input": "9b22/KlpnC_20447_a_B1-AR6-AMP_9b22/data/PSL-1011_1580_master.h5", "wavelength": 0.97856, "ref": {"sg": "P 1 21 1", "sgno": 4, "cell": [39.826, 92.659, 57.716, 90.0, 91.68, 90.0], "dmin": 1.3}, "tags": ["h5", "monoclinic"]},
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
"""paper_plots.py -- three battery figures of the open arm: R-free and high-resolution limit
|
||||
(deposition vs Rugnux) and the processing time per dataset.
|
||||
|
||||
tools/battery/paper_plots.py RUN [RUN ...] --out DIR [--time-runs RUN [RUN ...]] [--uncontended RUN]
|
||||
tools/battery/paper_plots.py RUN [RUN ...] --out DIR [--time-runs RUN [RUN ...]] [--time-arms ARM [ARM ...]]
|
||||
[--uncontended RUN]
|
||||
|
||||
RUN is a battery run directory (or its name under the site's runs_root); later runs fill in sets
|
||||
an earlier one lacks or that failed to run there (a `-redo` of CUDA casualties). --time-runs draws
|
||||
the time histogram from other runs than the two quality figures - a run whose sets had the GPU to
|
||||
themselves, where the runs for the quality figures had to share it. Only the open arm is read. Everything the figures are drawn from is written
|
||||
themselves, where the runs for the quality figures had to share it. The quality figures read only the
|
||||
open arm; --time-arms widens the time figures (e.g. `open inhouse`). Everything the figures are drawn from is written
|
||||
beside them, so a figure can be audited and redrawn: rfree.dat, dmin.dat (value pairs with the set
|
||||
and its beamline), time.dat (set, arm, seconds, images, other GPU processes), excluded.txt (every
|
||||
set left out, with the reason), summary.txt (the runs, their binary, the counts and the timing
|
||||
@@ -35,7 +37,8 @@ minute), and a run under --gpulock can share the GPU with other processes - `gpu
|
||||
them as seen just before the set started. Sets that ran with a shared GPU are drawn as the lighter
|
||||
top segment of their bar and are never dropped; with --uncontended RUN (a run of the same sets
|
||||
with the GPU to itself) summary.txt also reports how much slower they were than there. Sets that
|
||||
crashed have no WALL_TIME and go to excluded.txt.
|
||||
crashed have no WALL_TIME and go to excluded.txt. A fourth figure, time_zoom, is the same histogram
|
||||
over the first 60 s in 5 s bins, where most sets fall.
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
@@ -52,6 +55,8 @@ import gemmi
|
||||
RUNS_ROOT = "/data/battery/runs"
|
||||
RFREE_BAND = 0.05 # |R-free Rugnux - R-free depositor| counted as "within"
|
||||
TIME_BIN = 25 # s, histogram bin width
|
||||
ZOOM_MAX = 60 # s, range of the zoomed time histogram
|
||||
ZOOM_BIN = 5 # s, its bin width
|
||||
|
||||
# Sized and lettered to the IUCr artwork guide (journals.iucr.org/services/help/artwork/guide.html):
|
||||
# one column is 8.85 cm, lettering upright in a standard font (Helvetica, embedded by cairo) and in
|
||||
@@ -115,17 +120,29 @@ set label 5 "median %(tmed).0f s" at %(tmed).1f + 0.02 * %(thi)d, graph 0.93 lef
|
||||
bin(x) = (floor(x / %(bin)d) + 0.5) * %(bin)d
|
||||
plot "%(out)s/time.dat" using (bin($3)):(1) smooth frequency with boxes lc rgb "#8fc0e0" title "GPU shared", \
|
||||
"" using (bin($3)):($5 > 0 ? 0 : 1) smooth frequency with boxes lc rgb "#0072b2" title "GPU exclusive"
|
||||
|
||||
# --- the same over the first %(zhi)d s; sets beyond it are counted in the label, not drawn
|
||||
set output "%(out)s/time_zoom.pdf"
|
||||
set xrange [0:%(zhi)d]; set yrange [0:%(zyhi)d]
|
||||
set xtics 10; set ytics 5
|
||||
set boxwidth %(zbin)d absolute
|
||||
set label 11 "Datasets\nbeyond %(zhi)d s" at graph 0.64, graph 0.93 left front
|
||||
set label 12 "%(tn)d\n%(zbeyond)d" at graph 0.95, graph 0.93 right front
|
||||
set label 5 "median %(tmed).0f s" at %(tmed).1f + 0.02 * %(zhi)d, graph 0.80 left front
|
||||
zbin(x) = (floor(x / %(zbin)d) + 0.5) * %(zbin)d
|
||||
plot "%(out)s/time.dat" using (zbin($3)):($3 < %(zhi)d ? 1 : 0) smooth frequency with boxes lc rgb "#8fc0e0" title "GPU shared", \
|
||||
"" using (zbin($3)):($5 > 0 || $3 >= %(zhi)d ? 0 : 1) smooth frequency with boxes lc rgb "#0072b2" title "GPU exclusive"
|
||||
"""
|
||||
|
||||
|
||||
def load_rows(runs):
|
||||
"""Rows of the open arm, a later run's row replacing one that did not run."""
|
||||
def load_rows(runs, arms=("open",)):
|
||||
"""Rows of the given arms, a later run's row replacing one that did not run."""
|
||||
rows, manifests = {}, []
|
||||
for run in runs:
|
||||
path = run if os.path.isdir(run) else os.path.join(RUNS_ROOT, run)
|
||||
manifests.append((path, json.load(open(os.path.join(path, "manifest.json")))))
|
||||
for r in json.load(open(os.path.join(path, "results.json"))):
|
||||
if r["arm"] != "open":
|
||||
if r["arm"] not in arms:
|
||||
continue
|
||||
if r["set"] not in rows or rows[r["set"]].get("exit_code"):
|
||||
rows[r["set"]] = r
|
||||
@@ -167,6 +184,8 @@ def main():
|
||||
help="which REFMAC R-free pair to draw (default: refined, on the shared free set)")
|
||||
ap.add_argument("--time-runs", nargs="+", help="runs to draw the time histogram from "
|
||||
"(default: the runs above)")
|
||||
ap.add_argument("--time-arms", nargs="+", default=["open"],
|
||||
help="arms the time figures count (default: open)")
|
||||
ap.add_argument("--uncontended", help="a run of the same sets with the GPU to itself, to size the "
|
||||
"slowdown of the sets that shared it")
|
||||
a = ap.parse_args()
|
||||
@@ -177,7 +196,9 @@ def main():
|
||||
out = os.path.abspath(a.out)
|
||||
|
||||
rows, manifests = load_rows(a.runs)
|
||||
time_rows, time_manifests = load_rows(a.time_runs) if a.time_runs else (rows, [])
|
||||
time_rows, time_manifests = load_rows(a.time_runs or a.runs, a.time_arms)
|
||||
if not a.time_runs:
|
||||
time_manifests = []
|
||||
excluded, rfree, dmin, where, times = [], [], [], {}, []
|
||||
for s, r in sorted(time_rows.items()):
|
||||
if r.get("rugnux_wall_s") is None:
|
||||
@@ -237,11 +258,14 @@ def main():
|
||||
rlo=0.05 * math.floor((min(values) - 0.01) / 0.05), rhi=0.05 * math.ceil((max(values) + 0.01) / 0.05),
|
||||
dlo=0.5, dhi=8.0,
|
||||
bin=TIME_BIN, thi=TIME_BIN * math.ceil(max(secs) / TIME_BIN), tyhi=10 * math.ceil((tallest + 1) / 10),
|
||||
tmed=tmed, tn=len(times))
|
||||
tmed=tmed, tn=len(times),
|
||||
zhi=ZOOM_MAX, zbin=ZOOM_BIN, zbeyond=sum(t >= ZOOM_MAX for t in secs),
|
||||
zyhi=5 * math.ceil((max(collections.Counter(math.floor(t / ZOOM_BIN) for t in secs
|
||||
if t < ZOOM_MAX).values()) + 1) / 5))
|
||||
gp = os.path.join(out, "figures.gp")
|
||||
open(gp, "w").write(GNUPLOT % params)
|
||||
subprocess.run(["gnuplot", gp], check=True)
|
||||
for fig in ("rfree", "dmin", "time"): # the PNG is the PDF rasterised at 600 d.p.i.
|
||||
for fig in ("rfree", "dmin", "time", "time_zoom"): # the PNG is the PDF rasterised at 600 d.p.i.
|
||||
subprocess.run(["pdftoppm", "-r", "600", "-png", "-singlefile", os.path.join(out, fig + ".pdf"),
|
||||
os.path.join(out, fig)], check=True)
|
||||
|
||||
@@ -256,7 +280,7 @@ def main():
|
||||
f"d_min: {dn} structures, {dnf} facilities, {dnb} beamlines; Rugnux reaches further on "
|
||||
f"{deeper}\n")
|
||||
queue_wait = max(time_rows[s]["elapsed_s"] - t for s, _, t, _, _ in times)
|
||||
fh.write(f"time: {len(times)} open-arm sets; field rugnux_wall_s "
|
||||
fh.write(f"time: {len(times)} sets of arm(s) {', '.join(a.time_arms)}; field rugnux_wall_s "
|
||||
f"(rugnux's WALL_TIME over the whole invocation, not the runner's elapsed_s, which adds the wait "
|
||||
f"for a GPU slot: here at most {queue_wait:.1f} s)\n"
|
||||
f" median {tmed:.1f} s, mean {sum(secs) / len(secs):.1f} s, p10 {percentile(secs, 0.1):.1f}, "
|
||||
|
||||
Reference in New Issue
Block a user