diff --git a/docs/RUGNUX_OVERVIEW.md b/docs/RUGNUX_OVERVIEW.md index 69cbdd48f..ac2bf8aa5 100644 --- a/docs/RUGNUX_OVERVIEW.md +++ b/docs/RUGNUX_OVERVIEW.md @@ -4,6 +4,8 @@ The map of a run, in the order it happens — one paragraph per stage, each link [data-analysis reference](CPU_DATA_ANALYSIS.md) where the algorithm lives. The walk-through is a rotation run with the defaults; stills differences are at the end. +![The rotation pipeline: pass 1 integrates half the sweep at the file's geometry, pass 2 the whole sweep at the refined one, and the better of the two goes on to the space group and the written merge](images/rugnux_pipeline.svg) + **Open the dataset.** The geometry, wavelength and goniometer come from the file ([What Rugnux reads](RUGNUX_FORMATS.md)). A goniometer axis makes it a rotation run, none makes it serial stills — nothing is asked of the user. @@ -74,3 +76,5 @@ subgroup ([§14](CPU_DATA_ANALYSIS_DECISIONS.md)). orientation-tilt post-refinement rather than a rocking curve, and a merohedral indexing ambiguity has to be broken per image, at integration time, against a reference or a model ([Advanced ▸ the indexing ambiguity](RUGNUX_ADVANCED.md#the-indexing-ambiguity)). + +![The stills pipeline: every image indexed on its own, the geometry refined first where the cell is known, and the crystals scaled and merged together](images/rugnux_pipeline_stills.svg) diff --git a/docs/images/rugnux_pipeline.svg b/docs/images/rugnux_pipeline.svg new file mode 100644 index 000000000..3f41c54df --- /dev/null +++ b/docs/images/rugnux_pipeline.svg @@ -0,0 +1,136 @@ + + + + +Per image(independent) + +Decode +bitshuffle/LZ4, CBF + +CUDA + +Azimuthal profile +noise, ice rings + +CUDA + +Spot finding +self-calibrating threshold + +CUDA + +Refine +beam centre, orientation + +autodiff + +Predict +Ewald crossing, partiality + +CUDA + +Integrate +2D profile fit + +Metal + +CUDA + + + + + + +Input +live, HDF5, CBF, SMV, marCCD + +Pre-scan +beam-stop shadow, beam centre, +spot widths + +tested + +CUDA + +frame sample + +every frame + +masks, spot widths + +Index +frame sample, 1D FFT + +tested + +autodiff + +CUDA + +Post-refine +distance, beam, cell, axis + +tested + +autodiff + +beam centre + +spots + +lattice + + +pass 1 + +pass 2, repeated while it gains +Whole dataset + +Scale & merge (P1) +3D combine of partials + +CUDA + +Symmetry +space group, twinning, tNCS + +tested + +Scale & merge +in space group, resolution cut +decay, absorption, modulation + +tested + +CUDA + +Structure & map +placement or MR, σA maps + +tested + +CUDA + +Report +MTZ, mmCIF, maps + + + + + +without a model + +pass 2, or pass 1 if it merges better + +CUDA +CUDA kernel when a GPU is present + +Metal +also an Apple Metal kernel + +autodiff +derivatives by automatic differentiation + +tested +accepted only after a test against the data + \ No newline at end of file diff --git a/docs/images/rugnux_pipeline_fig.py b/docs/images/rugnux_pipeline_fig.py new file mode 100644 index 000000000..875628fe8 --- /dev/null +++ b/docs/images/rugnux_pipeline_fig.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Extended Data Fig. 1 — Rugnux pipeline, per-image loop vs whole-dataset steps (minimal-text version).""" + +# Two figures in one style: the rotation path (rugnux_pipeline.svg) and the stills path +# (rugnux_pipeline_stills.svg), both written next to this script. + +import os + +W, H = 1600, 990 +BLUE, TEAL, PURPLE, ORANGE, INK = "#2F6FDE", "#1A8A7D", "#8E44AD", "#E8740C", "#222222" +PANEL_FILL, PANEL_STROKE = "#EEF3FC", "#B9CBEF" +GPU_BG, TEST_BG, METAL_BG, AD_BG = "#2F6FDE", "#1DB38B", "#555555", "#B7791F" +GREY = "#6B6B6B" + +out = [] +def a(s): out.append(s) + +def start(): + out.clear() + a(f'') + a('' + '' + '' + '') + a(f'') + +def finish(name): + a('') + open(os.path.join(os.path.dirname(os.path.abspath(__file__)), name), "w").write("\n".join(out)) + +def badge(x, y, w, bg, text): + a(f'') + a(f'{text}') + +BADGES = [("tested", 62, TEST_BG), ("autodiff", 68, AD_BG), ("Metal", 52, METAL_BG), ("CUDA", 52, GPU_BG)] + +def box(x, y, w, h, title, body, color, gpu=False, tested=False, metal=False, autodiff=False, dashed=False, ts=20, bs=15): + dash = ' stroke-dasharray="9,6"' if dashed else '' + a(f'') + a(f'{title}') + if body: + lines = body if isinstance(body, list) else [body] + for i, ln in enumerate(lines): + a(f'{ln}') + bx = x + w - 10 + for (text, bw, bg), on in zip(BADGES, (tested, autodiff, metal, gpu)): + if on: + bx -= bw + badge(bx, y+h-28, bw, bg, text) + bx -= 8 + +def path(d, dashed=False): + dash = ' stroke-dasharray="8,6"' if dashed else '' + a(f'') + +def label(x, y, text, anchor="middle", size=14): + a(f'{text}') + +def panel(x0, x1, y0, y1, tx): + a(f'') + a(f'Per image(independent)') + +def legend(): + LY = H-40 + lx = 60 + for text, bw, bg, desc, step in [("CUDA", 52, GPU_BG, "CUDA kernel when a GPU is present", 330), + ("Metal", 52, METAL_BG, "also an Apple Metal kernel", 280), + ("autodiff", 68, AD_BG, "derivatives by automatic differentiation", 370), + ("tested", 62, TEST_BG, "accepted only after a test against the data", 0)]: + badge(lx, LY-14, bw, bg, text) + a(f'{desc}') + lx += step + +# ---------- geometry +BW, BH = 220, 105 +PY0, PY1 = 300, 660 # loop panel +R1Y, R2Y = 350, 520 # rows inside panel +LX = 322 # loop-return / pass-2 column +IX, IY, IW, IH = 60, 150, 220, 80 # input +SY = 120 # pre-scan row +QY, QBW, QBH = 770, 230, BH + 22 # whole-dataset row +QX = [300, 555, 810, 1065, 1320] +OUTY = 710 + +def dataset_row(merge_p1, symmetry, merge, gpu): + a(f'Whole dataset') + box(QX[0], QY, QBW, QBH, "Scale & merge (P1)", merge_p1, TEAL, gpu=gpu) + box(QX[1], QY, QBW, QBH, "Symmetry", symmetry, TEAL, tested=True, bs=14) + box(QX[2], QY, QBW, QBH, "Scale & merge", merge, TEAL, gpu=gpu, tested=gpu, bs=14) + box(QX[3], QY, QBW, QBH, "Structure & map", "placement or MR, σA maps", PURPLE, gpu=True, tested=True, dashed=True, bs=14) + box(QX[4], QY, QBW, QBH, "Report", "MTZ, mmCIF, maps", ORANGE) + for i in range(4): + path(f"M{QX[i]+QBW},{QY+52} L{QX[i+1]-2},{QY+52}") + # without a model the merge goes straight to the report + path(f"M{QX[2]+QBW-40},{QY} L{QX[2]+QBW-40},{QY-26} L{QX[4]+40},{QY-26} L{QX[4]+40},{QY-2}") + label(QX[3]+QBW/2, QY-34, "without a model", size=13) + + +def rotation(): + start() + PX0, PX1 = 300, 1090 + C = [360, 610, 860] # columns inside panel + + # ---------- loop panel + panel(PX0, PX1, PY0, PY1, C[0]) + box(C[0], R1Y, BW, BH, "Decode", "bitshuffle/LZ4, CBF", BLUE, gpu=True) + box(C[1], R1Y, BW, BH, "Azimuthal profile", "noise, ice rings", BLUE, gpu=True) + box(C[2], R1Y, BW, BH, "Spot finding", "self-calibrating threshold", BLUE, gpu=True) + box(C[0], R2Y, BW, BH, "Refine", "beam centre, orientation", BLUE, autodiff=True) + box(C[1], R2Y, BW, BH, "Predict", "Ewald crossing, partiality", BLUE, gpu=True) + box(C[2], R2Y, BW, BH, "Integrate", "2D profile fit", BLUE, gpu=True, metal=True) + + for r in (R1Y, R2Y): + path(f"M{C[0]+BW},{r+52} L{C[1]-2},{r+52}") + path(f"M{C[1]+BW},{r+52} L{C[2]-2},{r+52}") + + # loop return: Integrate bottom -> panel bottom bus -> left column -> Decode left edge + path(f"M{C[2]+90},{R2Y+BH} L{C[2]+90},{PY1-22} L{LX},{PY1-22} L{LX},{R1Y+52} L{C[0]-2},{R1Y+52}") + + # ---------- Input + box(IX, IY, IW, IH, "Input", "live, HDF5, CBF, SMV, marCCD", INK, bs=14) + + # ---------- Pre-scan + SX = 560 + box(SX, SY, 240, BH+22, "Pre-scan", ["beam-stop shadow, beam centre,", "spot widths"], TEAL, gpu=True, tested=True) + + path(f"M{IX+IW},{IY+40} L{SX-2},{IY+40}") + label((IX+IW+SX)/2, IY+32, "frame sample", size=13) + path(f"M{IX+IW/2},{IY+IH} L{IX+IW/2},{IY+IH+40} L{C[0]+110},{IY+IH+40} L{C[0]+110},{PY0-2}") + label(LX+12, IY+IH+32, "every frame", anchor="start", size=13) + path(f"M{SX+120},{SY+BH+22} L{SX+120},{PY0-2}") + label(SX+132, SY+BH+22+28, "masks, spot widths", anchor="start", size=13) + + # ---------- Index and Post-refine + DX, DW = 1200, 250 + box(DX, R1Y, DW, BH, "Index", "frame sample, 1D FFT", TEAL, gpu=True, autodiff=True, tested=True) + box(DX, R2Y, DW, BH, "Post-refine", "distance, beam, cell, axis", TEAL, autodiff=True, tested=True) + + # the beam centre the pre-scan measured is a hypothesis: the first pass indexes at both centres + path(f"M{SX+240},{SY+60} L{DX+DW/2},{SY+60} L{DX+DW/2},{R1Y-2}") + label((SX+240+DX+DW/2)/2, SY+52, "beam centre", size=13) + + path(f"M{C[2]+BW},{R1Y+40} L{DX-2},{R1Y+40}") + label((C[2]+BW+DX)/2, R1Y+32, "spots", size=13) + BUSY = 480 + path(f"M{DX+40},{R1Y+BH} L{DX+40},{BUSY} L{C[0]+110},{BUSY} L{C[0]+110},{R2Y-2}") + label(DX+52, BUSY-8, "lattice", anchor="start", size=13) + path(f"M{C[2]+110},{R1Y+BH} L{C[2]+110},{BUSY-2}") + + path(f"M{C[2]+BW},{R2Y+52} L{DX-2},{R2Y+52}") + label((C[2]+BW+DX)/2, R2Y+44, "pass 1", size=13) + + TOPY = 80 + path(f"M{DX+DW},{R2Y+52} L{DX+DW+40},{R2Y+52} L{DX+DW+40},{TOPY} L{LX},{TOPY} L{LX},{R1Y+50}", dashed=True) + label(900, TOPY-8, "pass 2, repeated while it gains", size=14) + + # ---------- Whole-dataset chain + dataset_row("3D combine of partials", "space group, twinning, tNCS", + ["in space group, resolution cut", "decay, absorption, modulation"], gpu=True) + path(f"M{C[2]+160},{R2Y+BH} L{C[2]+160},{OUTY} L{QX[0]+120},{OUTY} L{QX[0]+120},{QY-2}") + label(QX[0]+135, OUTY-8, "pass 2, or pass 1 if it merges better", anchor="start", size=13) + + legend() + finish("rugnux_pipeline.svg") + + +def stills(): + start() + PX0, PX1 = 300, 1340 + C = [360, 610, 860, 1110] # columns inside panel + + # ---------- loop panel: the first row left to right, the second back right to left + panel(PX0, PX1, PY0, PY1, C[0]) + box(C[0], R1Y, BW, BH, "Decode", "bitshuffle/LZ4, CBF", BLUE, gpu=True) + box(C[1], R1Y, BW, BH, "Azimuthal profile", "noise per ring", BLUE, gpu=True) + box(C[2], R1Y, BW, BH, "Spot finding", "self-calibrating threshold", BLUE, gpu=True) + box(C[3], R1Y, BW, BH, "Index", "1D FFT, or ffbidx with a cell", BLUE, gpu=True, tested=True, bs=14) + box(C[3], R2Y, BW, BH, "Refine", "beam, orientation, cell", BLUE, autodiff=True) + box(C[2], R2Y, BW, BH, "Predict", "excitation-error cutoff", BLUE, gpu=True) + box(C[1], R2Y, BW, BH, "Integrate", "2D profile fit", BLUE, gpu=True, metal=True) + + for i in range(3): + path(f"M{C[i]+BW},{R1Y+52} L{C[i+1]-2},{R1Y+52}") + path(f"M{C[3]+BW/2},{R1Y+BH} L{C[3]+BW/2},{R2Y-2}") + path(f"M{C[3]},{R2Y+52} L{C[2]+BW+2},{R2Y+52}") + path(f"M{C[2]},{R2Y+52} L{C[1]+BW+2},{R2Y+52}") + + # loop return: Integrate left edge -> left column -> Decode left edge + path(f"M{C[1]},{R2Y+52} L{LX},{R2Y+52} L{LX},{R1Y+52} L{C[0]-2},{R1Y+52}") + + # ---------- Input, geometry refinement, pre-scan + box(IX, IY, IW, IH, "Input", "HDF5, CBF, SMV, marCCD", INK, bs=14) + GX, SX = 560, 900 + box(GX, SY, 240, BH+22, "Geometry", ["with a known cell:", "beam, distance, cell"], TEAL, autodiff=True, dashed=True) + box(SX, SY, 240, BH+22, "Pre-scan", ["beam-stop shadow,", "bad pixels"], TEAL, gpu=True) + + path(f"M{IX+IW},{IY+40} L{GX-2},{IY+40}") + label((IX+IW+GX)/2, IY+32, "strong frames", size=13) + path(f"M{GX+240},{IY+40} L{SX-2},{IY+40}") + path(f"M{IX+IW/2},{IY+IH} L{IX+IW/2},{IY+IH+40} L{C[0]+110},{IY+IH+40} L{C[0]+110},{PY0-2}") + label(LX+12, IY+IH+32, "every frame", anchor="start", size=13) + path(f"M{GX+120},{SY+BH+22} L{GX+120},{PY0-2}") + label(GX+132, SY+BH+22+28, "geometry", anchor="start", size=13) + path(f"M{SX+120},{SY+BH+22} L{SX+120},{PY0-2}") + label(SX+132, SY+BH+22+28, "masks", anchor="start", size=13) + + # ---------- Whole-dataset chain + dataset_row(["consensus cell,", "per-image scale"], "space group, twinning, tNCS", + ["in space group, resolution cut", "partiality, error model"], gpu=False) + path(f"M{C[1]+110},{R2Y+BH} L{C[1]+110},{OUTY} L{QX[0]+120},{OUTY} L{QX[0]+120},{QY-2}") + label(QX[0]+135, OUTY-8, "every crystal", anchor="start", size=13) + + legend() + finish("rugnux_pipeline_stills.svg") + + +rotation() +stills() +print("ok") diff --git a/docs/images/rugnux_pipeline_stills.svg b/docs/images/rugnux_pipeline_stills.svg new file mode 100644 index 000000000..87b93a83c --- /dev/null +++ b/docs/images/rugnux_pipeline_stills.svg @@ -0,0 +1,120 @@ + + + + +Per image(independent) + +Decode +bitshuffle/LZ4, CBF + +CUDA + +Azimuthal profile +noise per ring + +CUDA + +Spot finding +self-calibrating threshold + +CUDA + +Index +1D FFT, or ffbidx with a cell + +tested + +CUDA + +Refine +beam, orientation, cell + +autodiff + +Predict +excitation-error cutoff + +CUDA + +Integrate +2D profile fit + +Metal + +CUDA + + + + + + + + +Input +HDF5, CBF, SMV, marCCD + +Geometry +with a known cell: +beam, distance, cell + +autodiff + +Pre-scan +beam-stop shadow, +bad pixels + +CUDA + +strong frames + + +every frame + +geometry + +masks +Whole dataset + +Scale & merge (P1) +consensus cell, +per-image scale + +Symmetry +space group, twinning, tNCS + +tested + +Scale & merge +in space group, resolution cut +partiality, error model + +Structure & map +placement or MR, σA maps + +tested + +CUDA + +Report +MTZ, mmCIF, maps + + + + + +without a model + +every crystal + +CUDA +CUDA kernel when a GPU is present + +Metal +also an Apple Metal kernel + +autodiff +derivatives by automatic differentiation + +tested +accepted only after a test against the data + \ No newline at end of file